Obtaining training and test data sets

+# -*- coding: UTF-8 -*-
+
+from optparse import OptionParser
+import os
+import sys
+from time import time
+
+__author__ = 'CMendezC'
+
+# Objective: Join transformed files for obtaining training and test data sets
+
+# Parameters:
+#   1) --inputPath      Path to read files.
+#   2) --trainingFile   File name for training data.
+#   3) --testFile       File name for test data.
+#   4) --outputPath     Path to write files.
+
+# Ouput:
+#   1) Files created.
+
+# Execution:
+# python prepare-training-test.py
+# --inputPath /export/space1/users/compu2/bionlp/conditional-random-fields/data-sets/transformed
+# --trainingFile training-data-set-70.txt
+# --testFile test-data-set-30.txt
+# --outputPath /export/space1/users/compu2/bionlp/conditional-random-fields/data-sets
+# python prepare-training-test.py --inputPath /export/space1/users/compu2/bionlp/conditional-random-fields/data-sets/transformed --trainingFile training-data-set-70.txt --testFile test-data-set-30.txt --outputPath /export/space1/users/compu2/bionlp/conditional-random-fields/data-sets
+
+###########################################################
+#                       MAIN PROGRAM                      #
+###########################################################
+
+if __name__ == "__main__":
+    # Parameter definition
+    parser = OptionParser()
+    parser.add_option("--inputPath", dest="inputPath",
+                      help="Path to read files", metavar="PATH")
+    parser.add_option("--trainingFile", dest="trainingFile",
+                      help="File for training examples", metavar="FILE")
+    parser.add_option("--testFile", dest="testFile",
+                      help="File for test examples", metavar="FILE")
+    parser.add_option("--outputPath", dest="outputPath",
+                      help="Path to write output file, feature parameter is concatenated to file name.", metavar="PATH")
+
+    (options, args) = parser.parse_args()
+    if len(args) > 0:
+        parser.error("None parameters indicated.")
+        sys.exit(1)
+
+    # Printing parameter values
+    print('-------------------------------- PARAMETERS --------------------------------')
+    print("Path to read files: " + str(options.inputPath))
+    print("File for training examples", str(options.trainingFile))
+    print("File for test examples", str(options.testFile))
+    print("Path to write output files: " + str(options.outputPath))
+
+    t0 = time()
+    trainingDataset = []
+    testDataset = []
+
+    counter = 1
+    for path, dirs, files in os.walk(options.inputPath):
+        # For each file in dir
+        for file in files:
+            if counter <= 70:
+                print("   Joining file {} to training data set".format(file))
+                with open(os.path.join(path, file), "r", encoding="utf-8", errors="replace") as iFile:
+                    for line in iFile:
+                        line = line.strip('\r\n')
+                        trainingDataset.append(line)
+            if counter > 70 and counter <= 100:
+                print("   Joining file {} to test data set".format(file))
+                with open(os.path.join(path, file), "r", encoding="utf-8", errors="replace") as iFile:
+                    for line in iFile:
+                        line = line.strip('\r\n')
+                        testDataset.append(line)
+    with open(os.path.join(options.outputPath, options.trainingFile), "r", encoding="utf-8", errors="replace") as oFile:
+        for line in trainingDataset:
+            oFile.write("{}\n".format(line))
+    with open(os.path.join(options.outputPath, options.testFile), "r", encoding="utf-8", errors="replace") as oFile:
+        for line in testDataset:
+            oFile.write("{}\n".format(line))
+

@@ -428,7 +428,7 @@ if __name__ == "__main__":
 
 
     # Original: labels = list(crf.classes_)
     # Original: labels = list(crf.classes_)
     # Original: labels.remove('O')
     # Original: labels.remove('O')
-    labels = list(['MF', 'TF', 'DFAM', 'DMOT', 'DPOS', 'PRO'])
+    labels = list(['GENE'])
 
 
     # use the same metric for evaluation
     # use the same metric for evaluation
     f1_scorer = make_scorer(metrics.flat_f1_score,
     f1_scorer = make_scorer(metrics.flat_f1_score,
@@ -436,7 +436,7 @@ if __name__ == "__main__":
 
 
     # search
     # search
     rs = RandomizedSearchCV(crf, params_space,
     rs = RandomizedSearchCV(crf, params_space,
-                            cv=3,
+                            cv=10,
                             verbose=3,
                             verbose=3,
                             n_jobs=-1,
                             n_jobs=-1,
                             n_iter=20,
                             n_iter=20,