Conditional Random Fields

+# -*- coding: UTF-8 -*-
+
+import os
+from itertools import chain
+from optparse import OptionParser
+from time import time
+from collections import Counter
+import re
+
+import nltk
+import sklearn
+import scipy.stats
+import sys
+
+from sklearn.externals import joblib
+from sklearn.metrics import make_scorer
+from sklearn.cross_validation import cross_val_score
+from sklearn.grid_search import RandomizedSearchCV
+
+import sklearn_crfsuite
+from sklearn_crfsuite import scorers
+from sklearn_crfsuite import metrics
+
+from nltk.corpus import stopwords
+
+
+# Objective
+# Training and evaluation of CRFs with sklearn-crfsuite.
+#
+# Input parameters
+# --inputPath=PATH    Path of training and test data set
+# --trainingFile        File with training data set
+# --testFile        File with test data set
+# --outputPath=PATH    Output path to place output files
+# --filteringStopWords   Filtering stop words
+# --excludeSymbols      Filtering punctuation marks
+
+# Output
+# 1) Best model
+
+# Examples
+# python3.4 training-validation-v1.py
+# --inputPath /export/space1/users/compu2/bionlp/conditional-random-fields/data-sets
+# --trainingFile training-data-set-70.txt
+# --testFile test-data-set-30.txt
+# --outputPath /export/space1/users/compu2/bionlp/conditional-random-fields
+# python3.4 training-validation-v1.py --inputPath /export/space1/users/compu2/bionlp/conditional-random-fields/data-sets --trainingFile training-data-set-70.txt --testFile test-data-set-30.txt --outputPath /export/space1/users/compu2/bionlp/conditional-random-fields
+
+#################################
+#           FUNCTIONS           #
+#################################
+def endsConLow(word):
+    miregex = re.compile(r'[^aeiouA-Z0-9]$')
+    if miregex.search(word):
+        return True
+    else:
+        return False
+
+def word2features(sent, i):
+    listElem = sent[i].split('|')
+    word = listElem[0]
+    lemma = listElem[1]
+    postag = listElem[2]
+
+    features = {
+        # Suffixes
+        'word[-3:]': word[-3:],
+        'word[-2:]': word[-2:],
+        'word[-1:]': word[-1:],
+        #'word.isupper()': word.isupper(),
+        'word': word,
+        'lemma': lemma,
+        'postag': postag,
+        'lemma[-3:]': lemma[-3:],
+        'lemma[-2:]': lemma[-2:],
+        'lemma[-1:]': lemma[-1:],
+        'word[:3]': word[:3],
+        'word[:2]': word[:2],
+        'word[:1]': word[:1],
+        'endsConLow()={}'.format(endsConLow(word)): endsConLow(word),
+    }
+    if i > 0:
+        listElem = sent[i - 1].split('|')
+        word1 = listElem[0]
+        lemma1 = listElem[1]
+        postag1 = listElem[2]
+        features.update({
+            '-1:word': word1,
+            '-1:lemma': lemma1,
+            '-1:postag': postag1,
+        })
+
+    if i < len(sent) - 1:
+        listElem = sent[i + 1].split('|')
+        word1 = listElem[0]
+        lemma1 = listElem[1]
+        postag1 = listElem[2]
+        features.update({
+            '+1:word': word1,
+            '+1:lemma': lemma1,
+            '+1:postag': postag1,
+        })
+
+    '''    
+    if i > 1:
+        listElem = sent[i - 2].split('|')
+        word2 = listElem[0]
+        lemma2 = listElem[1]
+        postag2 = listElem[2]
+        features.update({
+            '-2:word': word2,
+            '-2:lemma': lemma2,
+        })
+
+    if i < len(sent) - 2:
+        listElem = sent[i + 2].split('|')
+        word2 = listElem[0]
+        lemma2 = listElem[1]
+        postag2 = listElem[2]
+        features.update({
+            '+2:word': word2,
+            '+2:lemma': lemma2,
+        })
+
+    trigrams = False
+    if trigrams:
+        if i > 2:
+            listElem = sent[i - 3].split('|')
+            word3 = listElem[0]
+            lemma3 = listElem[1]
+            postag3 = listElem[2]
+            features.update({
+                '-3:word': word3,
+                '-3:lemma': lemma3,
+            })
+
+        if i < len(sent) - 3:
+            listElem = sent[i + 3].split('|')
+            word3 = listElem[0]
+            lemma3 = listElem[1]
+            postag3 = listElem[2]
+            features.update({
+                '+3:word': word3,
+                '+3:lemma': lemma3,
+            })
+    '''
+    return features
+
+
+def sent2features(sent):
+    return [word2features(sent, i) for i in range(len(sent))]
+
+
+def sent2labels(sent):
+    return [elem.split('|')[3] for elem in sent]
+
+
+def sent2tokens(sent):
+    return [token for token, postag, label in sent]
+
+
+def print_transitions(trans_features, f):
+    for (label_from, label_to), weight in trans_features:
+        f.write("{:6} -> {:7} {:0.6f}\n".format(label_from, label_to, weight))
+
+
+def print_state_features(state_features, f):
+    for (attr, label), weight in state_features:
+        f.write("{:0.6f} {:8} {}\n".format(weight, label, attr.encode("utf-8")))
+
+
+__author__ = 'CMendezC'
+
+##########################################
+#               MAIN PROGRAM             #
+##########################################
+
+if __name__ == "__main__":
+    # Defining parameters
+    parser = OptionParser()
+    parser.add_option("--inputPath", dest="inputPath",
+                      help="Path of training data set", metavar="PATH")
+    parser.add_option("--outputPath", dest="outputPath",
+                      help="Output path to place output files",
+                      metavar="PATH")
+    parser.add_option("--trainingFile", dest="trainingFile",
+                      help="File with training data set", metavar="FILE")
+    parser.add_option("--testFile", dest="testFile",
+                      help="File with test data set", metavar="FILE")
+    parser.add_option("--excludeStopWords", default=False,
+                      action="store_true", dest="excludeStopWords",
+                      help="Exclude stop words")
+    parser.add_option("--excludeSymbols", default=False,
+                      action="store_true", dest="excludeSymbols",
+                      help="Exclude punctuation marks")
+
+    (options, args) = parser.parse_args()
+    if len(args) > 0:
+        parser.error("Any parameter given.")
+        sys.exit(1)
+
+    print('-------------------------------- PARAMETERS --------------------------------')
+    print("Path of training data set: " + options.inputPath)
+    print("File with training data set: " + str(options.trainingFile))
+    print("Path of test data set: " + options.inputPath)
+    print("File with test data set: " + str(options.testFile))
+    print("Exclude stop words: " + str(options.excludeStopWords))
+    symbols = ['.', ',', ':', ';', '?', '!', '\'', '"', '<', '>', '(', ')', '-', '_', '/', '\\', '¿', '¡', '+', '{',
+               '}', '[', ']', '*', '%', '$', '#', '&', '°', '`', '...']
+    #print("Exclude symbols " + str(symbols) + ': ' + str(options.excludeSymbols))
+    print("Exclude symbols: " + str(options.excludeSymbols))
+
+    print('-------------------------------- PROCESSING --------------------------------')
+    print('Reading corpus...')
+    t0 = time()
+
+    sentencesTrainingData = []
+    sentencesTestData = []
+
+    stopwords = [word for word in stopwords.words('english')]
+
+    with open(os.path.join(options.inputPath, options.trainingFile), "r") as iFile:
+        for line in iFile.readlines():
+            listLine = []
+            line = line.strip('\n')
+            for token in line.split():
+                if options.excludeStopWords:
+                    listToken = token.split('|')
+                    lemma = listToken[1]
+                    if lemma in stopwords:
+                        continue
+                if options.excludeSymbols:
+                    listToken = token.split('|')
+                    lemma = listToken[1]
+                    if lemma in symbols:
+                        continue
+                listLine.append(token)
+            sentencesTrainingData.append(listLine)
+        print("   Sentences training data: " + str(len(sentencesTrainingData)))
+
+    with open(os.path.join(options.inputPath, options.testFile), "r") as iFile:
+        for line in iFile.readlines():
+            listLine = []
+            line = line.strip('\n')
+            for token in line.split():
+                if options.excludeStopWords:
+                    listToken = token.split('|')
+                    lemma = listToken[1]
+                    if lemma in stopwords:
+                        continue
+                if options.excludeSymbols:
+                    listToken = token.split('|')
+                    lemma = listToken[1]
+                    if lemma in symbols:
+                        continue
+                listLine.append(token)
+            sentencesTestData.append(listLine)
+        print("   Sentences test data: " + str(len(sentencesTestData)))
+
+    print("Reading corpus done in: %fs" % (time() - t0))
+
+    print(sent2features(sentencesTrainingData[0])[0])
+    print(sent2features(sentencesTestData[0])[0])
+    t0 = time()
+
+    X_train = [sent2features(s) for s in sentencesTrainingData]
+    y_train = [sent2labels(s) for s in sentencesTrainingData]
+
+    X_test = [sent2features(s) for s in sentencesTestData]
+    # print X_test
+    y_test = [sent2labels(s) for s in sentencesTestData]
+
+    # Fixed parameters
+    # crf = sklearn_crfsuite.CRF(
+    #     algorithm='lbfgs',
+    #     c1=0.1,
+    #     c2=0.1,
+    #     max_iterations=100,
+    #     all_possible_transitions=True
+    # )
+
+    # Hyperparameter Optimization
+    crf = sklearn_crfsuite.CRF(
+        algorithm='lbfgs',
+        max_iterations=100,
+        all_possible_transitions=True
+    )
+    params_space = {
+        'c1': scipy.stats.expon(scale=0.5),
+        'c2': scipy.stats.expon(scale=0.05),
+    }
+
+    # Original: labels = list(crf.classes_)
+    # Original: labels.remove('O')
+    labels = list(['GENE'])
+
+    # use the same metric for evaluation
+    f1_scorer = make_scorer(metrics.flat_f1_score,
+                            average='weighted', labels=labels)
+
+    # search
+    rs = RandomizedSearchCV(crf, params_space,
+                            cv=10,
+                            verbose=3,
+                            n_jobs=-1,
+                            n_iter=20,
+                            # n_iter=50,
+                            scoring=f1_scorer)
+    rs.fit(X_train, y_train)
+
+    # Fixed parameters
+    # crf.fit(X_train, y_train)
+
+    # Best hiperparameters
+    # crf = rs.best_estimator_
+    nameReport = options.trainingFile.replace('.txt', '.fStopWords_' + str(options.excludeStopWords) + '.fSymbols_' + str(
+        options.excludeSymbols) + '.txt')
+    with open(os.path.join(options.outputPath, "reports", "report_" + nameReport), mode="w") as oFile:
+        oFile.write("********** TRAINING AND TESTING REPORT **********\n")
+        oFile.write("Training file: " + options.trainingFile + '\n')
+        oFile.write('\n')
+        oFile.write('best params:' + str(rs.best_params_) + '\n')
+        oFile.write('best CV score:' + str(rs.best_score_) + '\n')
+        oFile.write('model size: {:0.2f}M\n'.format(rs.best_estimator_.size_ / 1000000))
+
+    print("Training done in: %fs" % (time() - t0))
+    t0 = time()
+
+    # Update best crf
+    crf = rs.best_estimator_
+
+    # Saving model
+    print("     Saving training model...")
+    t1 = time()
+    nameModel = options.trainingFile.replace('.txt', '.fStopWords_' + str(options.excludeStopWords) + '.fSymbols_' + str(
+        options.excludeSymbols) + '.mod')
+    joblib.dump(crf, os.path.join(options.outputPath, "models", nameModel))
+    print("        Saving training model done in: %fs" % (time() - t1))
+
+    # Evaluation against test data
+    y_pred = crf.predict(X_test)
+    print("*********************************")
+    name = options.trainingFile.replace('.txt', '.fStopWords_' + str(options.excludeStopWords) + '.fSymbols_' + str(
+        options.excludeSymbols) + '.txt')
+    with open(os.path.join(options.outputPath, "reports", "y_pred_" + name), "w") as oFile:
+        for y in y_pred:
+            oFile.write(str(y) + '\n')
+
+    print("*********************************")
+    name = options.trainingFile.replace('.txt', '.fStopWords_' + str(options.excludeStopWords) + '.fSymbols_' + str(
+        options.excludeSymbols) + '.txt')
+    with open(os.path.join(options.outputPath, "reports", "y_test_" + name), "w") as oFile:
+        for y in y_test:
+            oFile.write(str(y) + '\n')
+
+    print("Prediction done in: %fs" % (time() - t0))
+
+    # labels = list(crf.classes_)
+    # labels.remove('O')
+
+    with open(os.path.join(options.outputPath, "reports", "report_" + nameReport), mode="a") as oFile:
+        oFile.write('\n')
+        oFile.write("Flat F1: " + str(metrics.flat_f1_score(y_test, y_pred, average='weighted', labels=labels)))
+        oFile.write('\n')
+        # labels = list(crf.classes_)
+        sorted_labels = sorted(
+            labels,
+            key=lambda name: (name[1:], name[0])
+        )
+        oFile.write(metrics.flat_classification_report(
+            y_test, y_pred, labels=sorted_labels, digits=3
+        ))
+        oFile.write('\n')
+
+        oFile.write("\nTop likely transitions:\n")
+        print_transitions(Counter(crf.transition_features_).most_common(50), oFile)
+        oFile.write('\n')
+
+        oFile.write("\nTop unlikely transitions:\n")
+        print_transitions(Counter(crf.transition_features_).most_common()[-50:], oFile)
+        oFile.write('\n')
+
+        oFile.write("\nTop positive:\n")
+        print_state_features(Counter(crf.state_features_).most_common(200), oFile)
+        oFile.write('\n')
+
+        oFile.write("\nTop negative:\n")
+        print_state_features(Counter(crf.state_features_).most_common()[-200:], oFile)
+        oFile.write('\n')