Questions and Answers > NLP_Using_Python_ College of Engineering & Technology CSE AI

downloaded from

Scholarfriends

https://scholarfriends.com Downloaded by: writerforacademic@gmail.com | https://scholarfriends.com/singlePaper/276677/questions-and-answers-nlp-using-pythoncollege-of-engineering-technology-cse-ai Distribution of this document is illegal.

NLP Using Python 1. 1. HandsOn- Simple Operations with text a. Problem 1 #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd()+"/nltk_data" from nltk.corpus import gutenberg from nltk.text import Text # # Complete the 'calculateWordCounts' function below. # # # def calculateWordCounts(text): # Write your code here n_words = len(text) print(n_words) n_unique_words = len(set(text)) print(n_unique_words) word_coverage1 = math.floor(n_words / n_unique_words) print(word_coverage1) if __name__ == '__main__': text = input() if not os.path.exists(os.getcwd()+"/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) text = Text(gutenberg.words(text)) calculateWordCounts(text)

b. Problem 2 #!/bin/python3 import math

import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd() + "/nltk_data" from nltk.corpus import gutenberg from nltk.text import Text # # Complete the 'filterWords' function below. # # # def filterWords(text): # Write your code here unique_words = set(text) ing_words = [word for word in list(unique_words) if word.endswith('ing')] big_words = [word for word in list(text) if len(word) > 15 ] upper_words = [word for word in list(unique_words) if word.isupper()] return (ing_words, big_words, upper_words) if __name__ == '__main__': text = input() if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) text = Text(gutenberg.words(text)) ing_words, big_words, upper_words = filterWords(text) print(sorted(ing_words)) print(sorted(big_words)) print(sorted(upper_words))

c. Problem 3 #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd() + "/nltk_data" from nltk.corpus import gutenberg

from nltk.text import Text import nltk # # Complete the 'findWordFreq' function below. # # def findWordFreq(text, word): # Write your code here wordfreq = 0 for w in text: if(w == word): wordfreq = wordfreq+1 textfreq = nltk.FreqDist(word for word in list(text) if word.isalpha()) mostfreq = textfreq.max() return(wordfreq, mostfreq) if __name__ == '__main__': text = input() word = input() if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) text = Text(gutenberg.words(text)) word_freq, max_freq = findWordFreq(text, word) print(word_freq) print(max_freq)

2. Hands-On-Accessing Text Corpora a. Problem 1 #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd() + "/nltk_data" import nltk

from nltk.corpus import inaugural # # Complete the 'accessTextCorpora' function below. # # The function accepts following parameters: # 1. STRING fileid # 2. STRING word # def accessTextCorpora(fileid, word): # Write your code here file_words = inaugural.words(fileid) wordcoverage = int(len(file_words)/len(set(file_words))) ed_words = [words for words in set(file_words) if words.en dswith('ed')] textfreq2 = [word.lower() for word in file_words if word.i salpha()] textfreq = nltk.FreqDist(textfreq2) wordfreq = textfreq[word] return wordcoverage, ed_words, wordfreq if __name__ == '__main__': fileid = input() word = input() if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) word_coverage, ed_words, word_freq = accessTextCorpora(fil eid, word) print(word_coverage) print(sorted(ed_words)) print(word_freq)

b. Problem 2 #!/bin/python3 import math

import os import random import re import sys import nltk from nltk.corpus import PlaintextCorpusReader # # Complete the 'createUserTextCorpora' function below. # # The function accepts following parameters: # 1. STRING filecontent1 # 2. STRING filecontent2 # def createUserTextCorpora(filecontent1, filecontent2): # Write your code here with open(os.path.join('nltk_data/','content1.txt'),"w") a s file1: file1.write(filecontent1) file1.close() with open(os.path.join('nltk_data/','content2.txt'),"w") a s file2: file2.write(filecontent2) file2.close() text_corpus = PlaintextCorpusReader('nltk_data/','.*') no_of_words_corpus1 = len(text_corpus.words('content1.txt' )) no_of_words_corpus2 = len(text_corpus.words('content2.txt' )) no_of_unique_words_corpus1 = len(set(text_corpus.words('co ntent1.txt'))) no_of_unique_words_corpus2 = len(set(text_corpus.words('co ntent2.txt'))) return text_corpus,no_of_words_corpus1,no_of_unique_words_ corpus1,no_of_words_corpus2,no_of_unique_words_corpus2 if __name__ == '__main__': filecontent1 = input() filecontent2 = input()

path = os.path.join(os.getcwd(), "nltk_data") os.makedirs(path, exist_ok=True) for file in os.listdir(path): os.remove(path+"\\"+file)

text_corpus, no_of_words_corpus1, no_of_unique_words_corpu s1, no_of_words_corpus2, no_of_unique_words_corpus2 = createUs erTextCorpora(filecontent1, filecontent2) expected_corpus_files = ['content1.txt', 'content2.txt'] if type(text_corpus) == nltk.corpus.reader.plaintext.Plain textCorpusReader and sorted(list(text_corpus.fileids())) == ex pected_corpus_files: print(no_of_words_corpus1) print(no_of_unique_words_corpus1) print(no_of_words_corpus2) print(no_of_unique_words_corpus2)

3. Handson-Conditional Frequency distribution #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd()+"/nltk_data" import nltk from nltk.corpus import brown,stopwords # # Complete the 'calculateCFD' function below. # # The function accepts following parameters: # 1. STRING_ARRAY cfdconditions # 2. STRING_ARRAY cfdevents # def calculateCFD(cfdconditions, cfdevents): # Write your code here stopword = set(stopwords.words('english'))

cdev_cfd = nltk.ConditionalFreqDist([(genre, word.lower()) for genr e in brown.categories() for word in brown.words(categories=genre) if no t word.lower() in stopword]) cdev_cfd.tabulate(conditions = cfdconditions, samples = cfdevents) inged_cfd = [ (genre, word.lower()) for genre in brown.categories() for word in brown.words(categories=genre) if (word.lower().endswith('i ng') or word.lower().endswith('ed')) ] inged_cfd = [list(x) for x in inged_cfd] for wd in inged_cfd: if wd[1].endswith('ing') and wd[1] not in stopword: wd[1] = 'ing' elif wd[1].endswith('ed') and wd[1] not in stopword: wd[1] = 'ed' #print(inged_cfd) inged_cfd = nltk.ConditionalFreqDist(inged_cfd) #print(inged_cfd.conditions()) inged_cfd.tabulate(conditions=cfdconditions, samples = ['ed','ing'] ) if __name__ == '__main__': cfdconditions_count = int(input().strip()) cfdconditions = [] for _ in range(cfdconditions_count): cfdconditions_item = input() cfdconditions.append(cfdconditions_item) cfdevents_count = int(input().strip()) cfdevents = [] for _ in range(cfdevents_count): cfdevents_item = input() cfdevents.append(cfdevents_item) if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) calculateCFD(cfdconditions, cfdevents)

4. Hands-On Processing Raw text #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd() + "/nltk_data" import nltk import urllib import requests # # Complete the 'processRawText' function below. # # The function accepts STRING textURL as parameter. # def processRawText(textURL): # Write your code here response = requests.get(textURL) textcontent = response.text tokenizedword = nltk.tokenize.word_tokenize(textcontent) #Step 2 tokenizedlcwords = [x.lower() for x in list(tokenizedword) if x != ''] noofwords = len(tokenizedlcwords) noofunqwords = len(set(tokenizedlcwords)) wordcov = math.floor((noofwords)/(noofunqwords)) workfreq = nltk.FreqDist([word for word in (tokenizedlcwords) if wo rd.isalpha()]) maxfreq = workfreq.max() return (noofwords, noofunqwords, wordcov, maxfreq) if __name__ == '__main__': textURL = input()

if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) noofwords, noofunqwords, wordcov, maxfreq = processRawText(textURL) print(noofwords) print(noofunqwords) print(wordcov) print(maxfreq)

5. Bigrams and ngrams #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd() + "/nltk_data" import nltk from nltk.corpus import stopwords from nltk import ConditionalFreqDist # # Complete the 'performBigramsAndCollocations' function below. # # The function accepts following parameters: # 1. STRING textcontent # 2. STRING word # def performBigramsAndCollocations(textcontent, word): # Write your code here tokenizedword = nltk.regexp_tokenize(textcontent, pattern = r'\w *', gaps = False) tokenizedwords = [x.lower() for x in tokenizedword if x != ''] tokenizedwordsbigrams=nltk.bigrams(tokenizedwords) stop_words= stopwords.words('english') tokenizednonstopwordsbigrams=[(w1,w2) for w1 , w2 in tokenizedwo rdsbigrams if (w1 not in stop_words and w2 not in stop_words)]

cfd_bigrams=nltk.ConditionalFreqDist(tokenizednonstopwordsbigram s) mostfrequentwordafter=cfd_bigrams[word].most_common(3) tokenizedwords = nltk.Text(tokenizedwords) collocationwords = tokenizedwords.collocation_list() return mostfrequentwordafter ,collocationwords if __name__ == '__main__': textcontent = input() word = input() if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) mostfrequentwordafter, collocationwords = performBigramsAndCollo cations(textcontent, word) print(sorted(mostfrequentwordafter, key=lambda element: (element [1], element[0]), reverse=True)) print(sorted(collocationwords))

6. Stemming and Lemmatization #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd()+"/nltk_data" import nltk from nltk.corpus import stopwords # # Complete the 'performStemAndLemma' function below. # # The function accepts STRING textcontent as parameter. # def performStemAndLemma(textcontent):

# Write your code here #Step 1 tokenizedword = nltk.regexp_tokenize(textcontent, pattern = r'\w *', gaps = False) #Step 2 tokenizedwords = [y for y in tokenizedword if y != ''] unique_tokenizedwords = set(tokenizedwords) tokenizedwords = [x.lower() for x in unique_tokenizedwords if x != ''] #Step 3 #unique_tokenizedwords = set(tokenizedwords) stop_words = set(stopwords.words('english')) filteredwords = [] for x in tokenizedwords: if x not in stop_words: filteredwords.append(x) #Steps 4, 5 , 6 ps = nltk.stem.PorterStemmer() ls = nltk.stem.LancasterStemmer() wnl = nltk.stem.WordNetLemmatizer() porterstemmedwords =[] lancasterstemmedwords = [] lemmatizedwords = [] for x in filteredwords: porterstemmedwords.append(ps.stem(x)) lancasterstemmedwords.append(ls.stem(x)) lemmatizedwords.append(wnl.lemmatize(x)) return porterstemmedwords, lancasterstemmedwords, lemmatizedword s if __name__ == '__main__': textcontent = input() if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) porterstemmedwords, lancasterstemmedwords, lemmatizedwords = per formStemAndLemma(textcontent) print(sorted(porterstemmedwords)) print(sorted(lancasterstemmedwords)) print(sorted(lemmatizedwords))

7. POS Tagging #!/bin/python3 import math import os import random import re import sys import zipfile os.environ['NLTK_DATA'] = os.getcwd() + "/nltk_data" from nltk.corpus import brown import nltk

# # Complete the 'tagPOS' function below. # # The function accepts following parameters: # 1. STRING textcontent # 2. STRING taggedtextcontent # def tagPOS(textcontent, taggedtextcontent, defined_tags): # Write your code here text = nltk.word_tokenize(textcontent) nltk_pos_tags = nltk.pos_tag(text) #text2 = nltk.word_tokenize(taggedtextcontent) tagged_pos_tag = [nltk.tag.str2tuple(word) for word in taggedtextcontent.split()] tagger = nltk.UnigramTagger(model=defined_tags) unigram_pos_tag = tagger.tag(text) return nltk_pos_tags, tagged_pos_tag, unigram_pos_tag if __name__ == '__main__': textcontent = input() taggedtextcontent = input() if not os.path.exists(os.getcwd() + "/nltk_data"): with zipfile.ZipFile("nltk_data.zip", 'r') as zip_ref: zip_ref.extractall(os.getcwd()) defined_tags = dict(brown.tagged_words(tagset='universal')) nltk_pos_tags, tagged_pos_tag, unigram_pos_tag = tagPOS(textcontent, taggedtextcont ent, defined_tags) print(nltk_pos_tags) print(tagged_pos_tag) print(unigram_pos_tag)