bioNLP-UNAM / useless

Go to a project
Toggle navigation Toggle navigation pinning
iarroyof
a0f7b7ad 1 parent a5eecdbc

Final version for abstracts

Expand all
Inline Side-by-side
Showing 18 changed files with 0 additions and 493 deletions
#from pdb import set_trace as st
from sklearn.cross_validation import train_test_split as splitt
from sklearn.feature_extraction.text import TfidfVectorizer, HashingVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.svm import SVC
from sklearn import metrics
from sklearn.ensemble import (ExtraTreesClassifier, RandomForestClassifier,
AdaBoostClassifier, GradientBoostingClassifier)
from sklearn.grid_search import GridSearchCV
from sklearn.externals import joblib
import pandas as pd
from numpy import mean, std
class EstimatorSelectionHelper:
"http://www.codiply.com/blog/hyperparameter-grid-search-across-multiple-models-in-scikit-learn/"
def __init__(self, models, params):
if not set(models.keys()).issubset(set(params.keys())):
missing_params = list(set(models.keys()) - set(params.keys()))
raise ValueError("Some estimators are missing parameters: %s" % missing_params)
self.models = models
self.params = params
self.keys = models.keys()
self.grid_searches = {}
self.best_estimator = {}
def fit(self, X, y, cv=3, n_jobs=1, verbose=1, scoring=None, refit=False):
for key in self.keys:
print("Running GridSearchCV for %s." % key)
model = self.models[key]
params = self.params[key]
gs = GridSearchCV(model, params, cv=cv, n_jobs=n_jobs,
verbose=verbose, scoring=scoring, refit=refit)
gs.fit(X,y)
self.grid_searches[key] = gs
def score_summary(self, sort_by='mean_score'):
def row(key, scores, params, model):
d = {
'estimator': key,
'min_score': min(scores),
'max_score': max(scores),
'mean_score': mean(scores),
'std_score': std(scores),
'model': model
}
return pd.Series(dict(list(params.items()) + list(d.items())))
rows = [row(k, gsc.cv_validation_scores, gsc.parameters, m)
for k in self.keys
for gsc, m in zip(self.grid_searches[k].grid_scores_, self.grid_searches[k].best_estimator_)]
df = pd.concat(rows, axis=1).T.sort_values([sort_by], ascending=False)
columns = ['estimator', 'min_score', 'mean_score', 'max_score', 'std_score']
columns = columns + [c for c in df.columns if (c not in columns and c != 'model')]
self.best_estimator_ = df['model'][0]
return df[columns]
def get_abstracts(file_name, label):
f = open(file_name)
extract = {}
docs = []
empties = []
lines = f.readlines()
cpright = False
for i, ln in enumerate(lines):
if not ln.strip():
empties.append(i)
continue
elif ' doi: ' in ln:
for j in range(i, i + 10):
if not lines[j].strip():
title_idx = j + 1
break
continue
elif 'cpright ' in ln:
cpright = True
elif 'DOI: ' in ln:
if 'PMCID: ' in lines[i + 1]:
extract['pmid'] = int(lines[i + 2].strip().split()[1])
elif not 'PMCID: ' in lines[i + 1] and 'PMID: ' in lines[i + 1]:
extract['pmid'] = int(lines[i + 1].strip().split()[1])
if cpright:
get = slice(empties[-3], empties[-2])
cpright = False
else:
get = slice(empties[-2], empties[-1])
extract['body'] = " ".join(lines[get]).replace("\n", ' ').replace(" ", ' ')
title = []
for j in range(title_idx, title_idx + 5):
if lines[j].strip():
title.append(lines[j])
else:
break
extract['title'] = " ".join(title).replace("\n", ' ').replace(" ", ' ')
extract['topic'] = label
docs.append(extract)
empties = []
extract = {}
return docs
filename = "data/ecoli_abstracts/not_useful_abstracts.txt"
labels = ['useless', 'useful']
abstracs = get_abstracts(file_name=filename, label=labels[0])
filename = "data/ecoli_abstracts/useful_abstracts.txt"
abstracs += get_abstracts(file_name=filename, label=labels[1])
X = [x['body'] for x in abstracs]
y = [1 if x['topic'] == 'useful' else 0 for x in abstracs]
models1 = {
'ExtraTreesClassifier': ExtraTreesClassifier(),
'RandomForestClassifier': RandomForestClassifier(),
'AdaBoostClassifier': AdaBoostClassifier(),
'GradientBoostingClassifier': GradientBoostingClassifier(),
'SVC': SVC()
}
params1 = {
'ExtraTreesClassifier': {'n_estimators': [16, 32]},
'RandomForestClassifier': {'n_estimators': [16, 32]},
'AdaBoostClassifier': {'n_estimators': [16, 32]},
'GradientBoostingClassifier': {'n_estimators': [16, 32],
'learning_rate': [0.8, 1.0]},
'SVC': [
{'kernel': ['rbf'], 'C': [1, 10, 100, 150, 200, 300, 350, 400],
'gamma': [0.1, 0.01, 0.001, 0.0001, 0.00001]},
{'kernel': ['poly'], 'C': [1, 10, 100, 150, 200, 300, 350, 400],
'degree': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 23, 26],
'coef0': [0.1, 0.2,0.3,0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]}
]
}
clf = EstimatorSelectionHelper(models1, params1)
vectorizer = TfidfVectorizer(binary=True)
#ngram_range=(1, 3)
#)
#vectorizer = HashingVectorizer(non_negative=True)
print(vectorizer)
#svd = TruncatedSVD(n_components=200, random_state=42, n_iter=20)
X = vectorizer.fit_transform(X)
#X = svd.fit_transform(X)
#X_train, X_test, y_train, y_test = splitt(X, y, test_size=0.3, random_state=42)
#from sklearn.feature_selection import chi2, SelectKBest
#ch2 = SelectKBest(chi2, k=200)
#X_train = ch2.fit_transform(X_train, y_train)
#X_test = ch2.transform(X_test)
#clf = MultinomialNB(alpha=.01)
#clf = Classifier(n_jobs=-1, n_iter=100)
#st()
clf.fit(X, y, scoring='f1', n_jobs=-1)
#pred = clf.predict(X_test)
#print(metrics.f1_score(y_test, pred, average='macro'))
print(clf.score_summary(sort_by='min_score'))
joblib.dump(clf.best_estimator_, 'model/svm_model.pkl')
joblib.dump(vectorizer, 'model/tifidf_model.pkl')
#from pdb import set_trace as st
from sklearn.cross_validation import train_test_split as splitt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn import metrics
from sklearn.svm import SVC
import numpy as np
import argparse
import csv
import os
from sklearn.externals import joblib
from time import time
from scipy.stats import randint as sp_randint
from scipy.stats import expon
from sklearn.preprocessing import label_binarize
def get_abstracts(file_name, label):
f = open(file_name)
extract = {}
docs = []
empties = []
lines = f.readlines()
copyright = False
for i, ln in enumerate(lines):
if not ln.strip():
empties.append(i)
continue
elif ' doi: ' in ln:
for j in range(i, i + 10):
if not lines[j].strip():
title_idx = j + 1
break
continue
elif 'Copyright ' in ln or 'Publish' in ln or u'\N{COPYRIGHT SIGN}' in ln:
copyright = True
elif 'DOI: ' in ln:
if 'PMCID: ' in lines[i + 1]:
extract['pmid'] = int(lines[i + 2].strip().split()[1])
elif not 'PMCID: ' in lines[i + 1] and 'PMID: ' in lines[i + 1]:
extract['pmid'] = int(lines[i + 1].strip().split()[1])
if copyright:
get = slice(empties[-3], empties[-2])
copyright = False
else:
get = slice(empties[-2], empties[-1])
extract['body'] = " ".join(lines[get]).replace("\n", ' '
).replace(" ", ' ')
title = []
for j in range(title_idx, title_idx + 5):
if lines[j].strip():
title.append(lines[j])
else:
break
extract['title'] = " ".join(title).replace("\n", ' '
).replace(" ", ' ')
extract['topic'] = label
docs.append(extract)
empties = []
extract = {}
return docs
parser = argparse.ArgumentParser(
description="This script separates abstracts of biomedical papers that"
"report data from biomedical experiments from those that do not.")
parser.add_argument("--input", help="Input file containing the abstracts to"
"be predited.")
parser.add_argument("--classA", help="Input file containing the abstracts of"
"class A to be learned.")
parser.add_argument("--classB", help="Input file containing the abstracts of"
"class B to be learned.")
parser.add_argument("--out", help="Path to the output directory "
"(default='./filter_output')", default="filter_output")
parser.add_argument("--svcmodel", help="Path to custom pretrained svc model"
"(default='./model/svm_model.pkl')", default="model/svm_model.pkl")
args = parser.parse_args()
labels = {0: 'useless', 1: 'useful'}
vectorizer = TfidfVectorizer(binary=True)
print(vectorizer)
if args.classA and args.classA and not args.input:
f0 = open("model_params.conf")
n_iter_search = 10
params = [p for p in csv.DictReader(f0)]
f0.close()
names = list(params[0].keys())
model_params = {n: [] for n in names}
for n in names:
for d in params:
for k in d:
if k == n:
try:
model_params[n].append(float(d[k]))
except ValueError:
model_params[n].append(d[k])
model_params = {k: list(set(model_params[k])) for k in model_params}
abstracs = get_abstracts(file_name=args.classA, label=labels[0])
abstracs += get_abstracts(file_name=args.classB, label=labels[1])
tfidf_model = vectorizer.fit([x['body'] for x in abstracs])
X = vectorizer.transform([x['body'] for x in abstracs])
#y = [x['topic'] for x in abstracs]
y = [0 if x['topic'] == 'useless' else 1 for x in abstracs]
#X_train, X_test, y_train, y_test = splitt(X, y, test_size=0.3, random_state=42)
clf = SVC()#kernel='linear', C=100.0, gamma=0.0001)# degree=11, coef0=0.9)
clf = GridSearchCV(clf, cv=3,
param_grid=model_params,
# clf = RandomizedSearchCV(clf, param_distributions=model_params, cv=5, n_iter=n_iter_search,
n_jobs=-1, scoring='f1')
start = time()
clf.fit(X, y)
#clf.fit(X_train, y_train)
print("GridSearch took %.2f seconds for %d candidates"
" parameter settings." % ((time() - start), n_iter_search))
print(clf.best_estimator_)
print()
print(clf.best_score_)
#print(metrics.f1_score(clf.predict(X_test), y_test))
#joblib.dump(clf, 'model/svm_model.pkl')
joblib.dump(clf.best_estimator_, 'model/svm_model.pkl')
joblib.dump(tfidf_model, 'model/tfidf_model.pkl')
else:
clf = joblib.load(args.svcmodel)
vectorizer = joblib.load('model/tfidf_model.pkl')
abstracs = get_abstracts(file_name=args.input, label='unknown')
X = vectorizer.transform([x['body'] for x in abstracs])
classes = clf.predict(X)
if not os.path.exists(args.out):
os.makedirs(args.out)
# Writing predictions to output files
with open(args.out + "/" + labels[0] + ".out", 'w') as f0, \
open(args.out + "/" + labels[1] + ".out", 'w') as f1:
for c, a in zip(classes, abstracs):
if c == 0:
f0.write("%d\t%s\n" % (a['pmid'], a['body']))
elif c == 1:
f1.write("%d\t%s\n" % (a['pmid'], a['body']))
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
kernel,degree,coef0,C,gamma
linear,1,0.5,100,0.0
linear,1,0.5,10,0.0
linear,1,0.5,50,0.0
linear,1,0.5,100,0.0
linear,1,0.5,5,0.0
linear,1,0.5,150,0.0
linear,1,0.5,200,0.0
linear,1,0.5,300,0.0
linear,1,0.5,400,0.0
linear,1,0.5,1.0,0.0
linear,1,0.5,5.0,0.0
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
TfidfVectorizer(analyzer='word', binary=True, decode_error='strict',
dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',
lowercase=True, max_df=1.0, max_features=None, min_df=1,
ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,
stop_words=None, strip_accents=None, sublinear_tf=False,
token_pattern='(?u)\\b\\w\\w+\\b', tokenizer=None, use_idf=True,
vocabulary=None)
Running GridSearchCV for GradientBoostingClassifier.
Fitting 3 folds for each of 4 candidates, totalling 12 fits
Running GridSearchCV for AdaBoostClassifier.
Fitting 3 folds for each of 2 candidates, totalling 6 fits
Running GridSearchCV for ExtraTreesClassifier.
Fitting 3 folds for each of 2 candidates, totalling 6 fits
Running GridSearchCV for SVC.
Fitting 3 folds for each of 63 candidates, totalling 189 fits
Running GridSearchCV for RandomForestClassifier.
Fitting 3 folds for each of 2 candidates, totalling 6 fits
estimator min_score mean_score max_score std_score \
36 SVC 0.69697 0.702911 0.705882 0.00420147
66 SVC 0.69697 0.702911 0.705882 0.00420147
35 SVC 0.69697 0.702911 0.705882 0.00420147
37 SVC 0.69697 0.702911 0.705882 0.00420147
38 SVC 0.69697 0.702911 0.705882 0.00420147
39 SVC 0.69697 0.702911 0.705882 0.00420147
40 SVC 0.69697 0.702911 0.705882 0.00420147
41 SVC 0.69697 0.702911 0.705882 0.00420147
42 SVC 0.69697 0.702911 0.705882 0.00420147
43 SVC 0.69697 0.702911 0.705882 0.00420147
44 SVC 0.69697 0.702911 0.705882 0.00420147
45 SVC 0.69697 0.702911 0.705882 0.00420147
46 SVC 0.69697 0.702911 0.705882 0.00420147
47 SVC 0.69697 0.702911 0.705882 0.00420147
48 SVC 0.69697 0.702911 0.705882 0.00420147
49 SVC 0.69697 0.702911 0.705882 0.00420147
50 SVC 0.69697 0.702911 0.705882 0.00420147
51 SVC 0.69697 0.702911 0.705882 0.00420147
52 SVC 0.69697 0.702911 0.705882 0.00420147
53 SVC 0.69697 0.702911 0.705882 0.00420147
54 SVC 0.69697 0.702911 0.705882 0.00420147
55 SVC 0.69697 0.702911 0.705882 0.00420147
56 SVC 0.69697 0.702911 0.705882 0.00420147
57 SVC 0.69697 0.702911 0.705882 0.00420147
58 SVC 0.69697 0.702911 0.705882 0.00420147
59 SVC 0.69697 0.702911 0.705882 0.00420147
60 SVC 0.69697 0.702911 0.705882 0.00420147
61 SVC 0.69697 0.702911 0.705882 0.00420147
62 SVC 0.69697 0.702911 0.705882 0.00420147
63 SVC 0.69697 0.702911 0.705882 0.00420147
.. ... ... ... ... ...
12 SVC 0.69697 0.702911 0.705882 0.00420147
13 SVC 0.69697 0.702911 0.705882 0.00420147
14 SVC 0.69697 0.702911 0.705882 0.00420147
15 SVC 0.69697 0.702911 0.705882 0.00420147
16 SVC 0.69697 0.702911 0.705882 0.00420147
17 SVC 0.69697 0.702911 0.705882 0.00420147
26 SVC 0.69697 0.702911 0.705882 0.00420147
25 SVC 0.69697 0.702911 0.705882 0.00420147
30 SVC 0.69697 0.702911 0.705882 0.00420147
29 SVC 0.69697 0.702911 0.705882 0.00420147
28 SVC 0.69697 0.702911 0.705882 0.00420147
27 SVC 0.69697 0.702911 0.705882 0.00420147
19 SVC 0.69697 0.702911 0.705882 0.00420147
65 SVC 0.69697 0.702911 0.705882 0.00420147
24 SVC 0.69697 0.702911 0.705882 0.00420147
23 SVC 0.69697 0.702911 0.705882 0.00420147
22 SVC 0.69697 0.702911 0.705882 0.00420147
21 SVC 0.69697 0.702911 0.705882 0.00420147
18 SVC 0.686567 0.693502 0.69697 0.0049038
20 SVC 0.676923 0.691047 0.707692 0.0126874
7 ExtraTreesClassifier 0.619048 0.662524 0.688525 0.0309388
6 ExtraTreesClassifier 0.588235 0.611627 0.655738 0.0312098
1 GradientBoostingClassifier 0.577778 0.595982 0.610169 0.0135256
0 GradientBoostingClassifier 0.5 0.549894 0.596491 0.0394613
71 RandomForestClassifier 0.470588 0.557789 0.625 0.0646035
3 GradientBoostingClassifier 0.454545 0.548927 0.596491 0.0667386
2 GradientBoostingClassifier 0.439024 0.588593 0.701754 0.110305
5 AdaBoostClassifier 0.411765 0.489657 0.618182 0.0915596
4 AdaBoostClassifier 0.4 0.54013 0.655172 0.105673
72 RandomForestClassifier 0.380952 0.504177 0.631579 0.10236
C degree gamma kernel learning_rate n_estimators
36 100 6 NaN poly NaN NaN
66 200 NaN 0.0001 sigmoid NaN NaN
35 100 5 NaN poly NaN NaN
37 150 2 NaN poly NaN NaN
38 150 3 NaN poly NaN NaN
39 150 4 NaN poly NaN NaN
40 150 5 NaN poly NaN NaN
41 150 6 NaN poly NaN NaN
42 200 2 NaN poly NaN NaN
43 200 3 NaN poly NaN NaN
44 200 4 NaN poly NaN NaN
45 200 5 NaN poly NaN NaN
46 200 6 NaN poly NaN NaN
47 300 2 NaN poly NaN NaN
48 300 3 NaN poly NaN NaN
49 300 4 NaN poly NaN NaN
50 300 5 NaN poly NaN NaN
51 300 6 NaN poly NaN NaN
52 400 2 NaN poly NaN NaN
53 400 3 NaN poly NaN NaN
54 400 4 NaN poly NaN NaN
55 400 5 NaN poly NaN NaN
56 400 6 NaN poly NaN NaN
57 1 NaN 0.001 sigmoid NaN NaN
58 1 NaN 0.0001 sigmoid NaN NaN
59 10 NaN 0.001 sigmoid NaN NaN
60 10 NaN 0.0001 sigmoid NaN NaN
61 100 NaN 0.001 sigmoid NaN NaN
62 100 NaN 0.0001 sigmoid NaN NaN
63 150 NaN 0.001 sigmoid NaN NaN
.. ... ... ... ... ... ...
12 100 NaN 0.001 rbf NaN NaN
13 100 NaN 0.0001 rbf NaN NaN
14 150 NaN 0.001 rbf NaN NaN
15 150 NaN 0.0001 rbf NaN NaN
16 200 NaN 0.001 rbf NaN NaN
17 200 NaN 0.0001 rbf NaN NaN
26 1 6 NaN poly NaN NaN
25 1 5 NaN poly NaN NaN
30 10 5 NaN poly NaN NaN
29 10 4 NaN poly NaN NaN
28 10 3 NaN poly NaN NaN
27 10 2 NaN poly NaN NaN
19 300 NaN 0.0001 rbf NaN NaN
65 200 NaN 0.001 sigmoid NaN NaN
24 1 4 NaN poly NaN NaN
23 1 3 NaN poly NaN NaN
22 1 2 NaN poly NaN NaN
21 400 NaN 0.0001 rbf NaN NaN
18 300 NaN 0.001 rbf NaN NaN
20 400 NaN 0.001 rbf NaN NaN
7 NaN NaN NaN NaN NaN 32
6 NaN NaN NaN NaN NaN 16
1 NaN NaN NaN NaN 0.8 32
0 NaN NaN NaN NaN 0.8 16
71 NaN NaN NaN NaN NaN 16
3 NaN NaN NaN NaN 1 32
2 NaN NaN NaN NaN 1 16
5 NaN NaN NaN NaN NaN 32
4 NaN NaN NaN NaN NaN 16
72 NaN NaN NaN NaN NaN 32
[73 rows x 11 columns]
This diff could not be displayed because it is too large.
28911122 ChIP-exo/nexus experiments rely on innovative modifications of the commonly used ChIP-seq protocol for high resolution mapping of transcription factor binding sites. Although many aspects of the ChIP-exo data analysis are similar to those of ChIP-seq, these high throughput experiments pose a number of unique quality control and analysis challenges. We develop a novel statistical quality control pipeline and accompanying R/Bioconductor package, ChIPexoQual, to enable exploration and analysis of ChIP-exo and related experiments. ChIPexoQual evaluates a number of key issues including strand imbalance, library complexity, and signal enrichment of data. Assessment of these features are facilitated through diagnostic plots and summary statistics computed over regions of the genome with varying levels of coverage. We evaluated our QC pipeline with both large collections of public ChIP-exo/nexus data and multiple, new ChIP-exo datasets from Escherichia coli. ChIPexoQual analysis of these datasets resulted in guidelines for using these QC metrics across a wide range of sequencing depths and provided further insights for modelling ChIP-exo data.
This diff is collapsed. Click to expand it.