Home » Football » England » Alan Turvey Trophy

Alan Turvey Trophy stats & predictions

Yesterday

Tomorrow

--<|repo_name|>xuekuiqin/Corpus-Based-NLP<|file_sep|>/task1/task1_1.py import pandas as pd import numpy as np import nltk import re def tokenize(sentence): sentence = re.sub(r'([^sw]|_)+', '', sentence) tokens = nltk.word_tokenize(sentence) return tokens def read_file(path): df = pd.read_csv(path) return df def count_words(tokens): word_count = dict() for token in tokens: if token not in word_count.keys(): word_count[token] = 0 word_count[token] += 1 return word_count def calculate_idf(word_count_dict): idf_dict = dict() num_documents = len(word_count_dict.keys()) print(num_documents) for word,count in word_count_dict.items(): idf_dict[word] = np.log(num_documents/count) return idf_dict if __name__ == "__main__": path = "D:/NLP/Corpus-Based-NLP/task1/lyrics.csv" df = read_file(path) #print(df.head(10)) word_count_dict = dict() #for index,row in df.iterrows(): #tokens = tokenize(row['lyrics']) #tokens_set = set(tokens) #for token in tokens_set: #if token not in word_count_dict.keys(): #word_count_dict[token] = set() #word_count_dict[token].add(index) ''' print(len(word_count_dict.keys())) print(word_count_dict["love"]) idf_dict = calculate_idf(word_count_dict) print(idf_dict["love"]) sorted_idf_list = sorted(idf_dict.items(), key=lambda x:x[1], reverse=True) print(sorted_idf_list[:100]) ''' <|file_sep|># Corpus-Based-NLP ## task1 ### task1_1.py * 数据来源：[The Million Song Dataset](https://labrosa.ee.columbia.edu/millionsong/) * 数据说明：文件包含两列，一列是歌曲名，另一列是歌词。 * 数据格式：CSV格式。 * 数据量：约400万行。 * 实现内容：计算所有歌曲的歌词中的每个单词的idf值，输出idf值最大的100个单词。 ### task1_2.py * 数据来源：[The Million Song Dataset](https://labrosa.ee.columbia.edu/millionsong/) * 数据说明：文件包含两列，一列是歌曲名，另一列是歌词。 * 数据格式：CSV格式。 * 数据量：约400万行。 * 实现内容：计算每个歌手的歌词中的每个单词的tf-idf值，输出tf-idf值最大的100个单词。 ### task1_2_test.py 测试代码，将所有歌曲按照不同的歌手进行划分。 ### task1_2_test_data.csv 测试数据集，仅包含10首不同歌手的歌曲。 ### task1_2_test_output.txt 测试结果。 ## task2 ### task2_1.py * 数据来源：[The Oxford English Dictionary (OED)](http://www.oed.com/) * 数据说明：文件包含两列，一列是单词，另一列是该单词的定义。 * 数据格式：CSV格式。 * 数据量：约130万行。 * 实现内容：计算所有单词定义中的每个单词的idf值，输出idf值最大的100个单词。 ### task2_2.py * 数据来源：[The Oxford English Dictionary (OED)](http://www.oed.com/) * 数据说明：文件包含两列，一列是单词，另一列是该单词的定义。 * 数据格式：CSV格式。 * 数据量：约130万行。 * 实现内容：计算每个部分中的单词定义中的每个单词的tf-idf值，输出tf-idf值最大的100个单词。 ### task2_2_test.py 测试代码，将所有定义按照不同部分进行划分。 ### task2_2_test_data.csv 测试数据集，仅包含10个部分及其定义。 ### task2_2_test_output.txt 测试结果。<|repo_name|>xuekuiqin/Corpus-Based-NLP<|file_sep|>/task2/task2_2.py import pandas as pd import numpy as np from nltk.stem import PorterStemmer def read_file(path): df = pd.read_csv(path) return df def tokenize(sentence): ps = PorterStemmer() tokens = sentence.split(" ") stemmed_tokens = [] for token in tokens: stemmed_token = ps.stem(token) stemmed_tokens.append(stemmed_token) return stemmed_tokens def count_words(df): part_word_count_dict = dict() if __name__ == "__main__": <|repo_name|>xuekuiqin/Corpus-Based-NLP<|file_sep|>/task1/task1_2.py import pandas as pd import numpy as np import nltk import re from collections import Counter def tokenize(sentence): sentence = re.sub(r'([^sw]|_)+', '', sentence) tokens = nltk.word_tokenize(sentence) return tokens def read_file(path): df = pd.read_csv(path) return df def count_words(tokens): word_count = dict() for token in tokens: if token not in word_count.keys(): word_count[token] = Counter() word_count[token][df['artist_name'][index]] += 1 return word_count def calculate_idf(word_count_dict): idf_dict = dict() num_documents = len(df['artist_name'].unique()) for word,counters in word_count_dict.items(): idf_sum = sum(counters.values()) idf_dict[word] = np.log(num_documents/idf_sum) return idf_dict if __name__ == "__main__": path = "D:/NLP/Corpus-Based-NLP/task1/lyrics.csv" df = read_file(path) word_count_list = [] for index,row in df.iterrows(): tokens_set = set(tokenize(row['lyrics'])) word_counts_per_sentence= count_words(tokens_set) word_count_list.append(word_counts_per_sentence) merged_word_counts= {} for word_counts_per_sentence in word_count_list: for word,counters in word_counts_per_sentence.items(): if not merged_word_counts.get(word): merged_word_counts[word] = counters else: for artist,count_per_artist in counters.items(): if merged_word_counts[word].get(artist): merged_word_counts[word][artist] += count_per_artist else: merged_word_counts[word][artist] = count_per_artist idf_dict=calculate_idf(merged_word_counts) def tf_idf_calculation(row): tfidf_list=[] for artist,index_row in row.items(): tfidf_list.append((artist,index_row['count']*idf_dict[index_row['word']])) return tfidf_list final_df=pd.DataFrame(columns=['artist_name','word','tfidf']) for word,counters in merged_word_counts.items(): word_series=pd.Series(counters) word_series.name=word word_series.index.name='artist_name' tfidf_list=tf_idf_calculation(word_series.to_frame()) for item in tfidf_list: temp_df=pd.DataFrame({'artist_name':[item[0]],'word':[word],'tfidf':[item[1]]}) final_df=final_df.append(temp_df) final_df.sort_values(by='tfidf',ascending=False,inplace=True) final_df.to_csv('output.csv',index=False)<|repo_name|>xuekuiqin/Corpus-Based-NLP<|file_sep|>/task2/task2_1.py import pandas as pd import numpy as np from nltk.stem import PorterStemmer def read_file(path): df=pd.read_csv(path) return df def tokenize(sentence): ps=PorterStemmer() tokens=sentence.split(" ") stemmed_tokens=[] for token in tokens: stemmed_token=ps.stem(token) stemmed_tokens.append(stemmed_token) return stemmed_tokens def count_words(df): word_counts=dict() for index,row in df.iterrows(): stemmed_tokens=tokenize(row['definition']) stemmed_tokens_set=set(stemmed_tokens) for token in stemmed_tokens_set: if not word_counts.get(token): word_counts[token]=0 word_counts[token]+=1 return word_counts def calculate_idf(word_counts): idf=dict() num_documents=len(df.index) for word,count in word_counts.items(): idf[word]=np.log(num_documents/count) return idf if __name__=="__main__": path="D:/NLP/Corpus-Based-NLP/task2/oed_dictionary_definitions.csv" df=read_file(path) word_counts=count_words(df) idfs=calculate_idf(word_counts) sorted_idfs=sorted(idfs.items(),key=lambda x:x[1],reverse=True) print(sorted_idfs[:100])<|file_sep|>#ifndef CHATROOM_H_ #define CHATROOM_H_ #include "User.h" #include "Message.h" #include "../Utilities/DataBase.h" #include "../Utilities/Parser.h" #include "../Utilities/StringHelper.h" #include "../DataStructures/List.h" #include "../DataStructures/Stack.h" #include "../DataStructures/Queue.h" class ChatRoom { private: int m_roomId; StringHelper m_name; List m_users; DataBase m_db; bool m_isPublic; bool m_isPrivate; public: enum ROOM_TYPE { PUBLIC_ROOM, PRIVATE_ROOM }; public: void createRoom(int roomId); void createRoom(int roomId, StringHelper name); void createRoom(int roomId, StringHelper name, bool isPublic); void createRoom(int roomId, StringHelper name, bool isPublic, DataBase &db); void deleteRoom(int roomId);