In [65]:
import pandas as pd
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.cluster import KMeans, SpectralClustering, AgglomerativeClustering
import matplotlib.pyplot as plt
from sklearn.metrics import silhouette_score, homogeneity_score
from scipy.cluster.hierarchy import dendrogram, linkage
from minisom import MiniSom
In [11]:
data_B = pd.read_csv('DATA_B.csv')
data_M = pd.read_csv('DATA_M.csv')
In [25]:
data_T = pd.read_csv('DATA_T.csv')
In [27]:
data = data_B.append(data_M).append(data_T)
In [28]:
data = data.iloc[:,1:]
In [33]:
data
Out[33]:
| ENSG00000000003 | ENSG00000000005 | ENSG00000000419 | ENSG00000000457 | ENSG00000000460 | ENSG00000000938 | ENSG00000000971 | ENSG00000001036 | ENSG00000001084 | ENSG00000001167 | ... | ENSGR0000237801 | ENSGR0000263835 | ENSGR0000263980 | ENSGR0000264510 | ENSGR0000264819 | ENSGR0000265350 | ENSGR0000265658 | ENSGR0000266731 | ENSGR0000270726 | class | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 1.021593 | 0.0 | 60.154811 | 33.748978 | 7.227303 | 152.713882 | 0.015776 | 10.608696 | 10.682172 | 279.354711 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0 |
| 1 | 0.414410 | 0.0 | 64.760742 | 35.746256 | 7.660886 | 158.551228 | 0.000000 | 12.130671 | 8.854435 | 238.094225 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0 |
| 2 | 0.431737 | 0.0 | 62.409309 | 35.692106 | 7.969141 | 164.157274 | 0.000000 | 12.706457 | 9.123782 | 235.266887 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0 |
| 3 | 0.443048 | 0.0 | 61.696916 | 36.244325 | 8.149770 | 169.366240 | 0.000000 | 13.005400 | 9.281539 | 214.822081 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0 |
| 4 | 0.413403 | 0.0 | 62.623580 | 37.138720 | 8.346329 | 167.355647 | 0.000000 | 13.203618 | 9.273433 | 220.142202 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 98 | 2.453327 | 0.0 | 81.517581 | 48.981071 | 4.735314 | 1.890310 | 0.000000 | 12.443251 | 12.866301 | 300.121065 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 2 |
| 99 | 2.464241 | 0.0 | 81.818407 | 48.803818 | 4.671851 | 1.818116 | 0.000000 | 11.523303 | 12.993471 | 305.594062 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 2 |
| 100 | 2.460441 | 0.0 | 83.380376 | 48.039287 | 4.732939 | 1.537744 | 0.000000 | 11.488891 | 12.854001 | 312.017960 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 2 |
| 101 | 2.349414 | 0.0 | 85.629051 | 47.141164 | 4.919891 | 1.598132 | 0.000000 | 11.400380 | 12.949581 | 324.559070 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 2 |
| 102 | 1.580929 | 0.0 | 91.098264 | 44.116596 | 5.056261 | 1.638622 | 0.000000 | 11.362697 | 13.431308 | 311.761240 | ... | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 2 |
315 rows × 57821 columns
In [34]:
X = data.iloc[:, :-1]
y = data.iloc[:, [-1]]
In [35]:
lda = LDA()
transf = lda.fit_transform(X, y.values.ravel())
In [38]:
transf
Out[38]:
array([[-236.59466997, -222.29934741],
[-236.69702612, -222.3743407 ],
[-236.63174941, -222.34590847],
[-236.66749046, -222.36024975],
[-236.60547597, -222.29004641],
[-236.60873676, -222.45850776],
[-236.75400066, -222.24932538],
[-236.48482133, -222.31381263],
[-236.76827545, -222.43621022],
[-236.62719447, -222.38905146],
[-236.6439594 , -222.22709842],
[-236.69569314, -222.40252857],
[-236.55774497, -222.31645638],
[-236.72977897, -222.41439657],
[-236.56650621, -222.38945675],
[-236.82828152, -222.1476838 ],
[-236.35457307, -222.16520694],
[-236.75687986, -222.82910048],
[-236.77815435, -222.04083244],
[-236.71389898, -222.71759388],
[-236.53648356, -221.95729112],
[-236.90799131, -222.18255108],
[-236.42856321, -222.76530225],
[-236.50265803, -221.56965756],
[-236.94798568, -223.62931669],
[-236.98482031, -222.22693764],
[-236.10366421, -221.64605317],
[-236.54249647, -222.2191927 ],
[-236.55598842, -222.80475795],
[-237.1720682 , -222.05801885],
[-235.7734045 , -221.59598057],
[-237.25437452, -223.31635497],
[-237.08475864, -221.92594057],
[-235.86082841, -222.64215585],
[-237.07815439, -222.3283986 ],
[-237.21539538, -222.93311554],
[-236.51648024, -222.24183033],
[-236.04578684, -222.32627103],
[-236.75214991, -222.02907071],
[-235.98154641, -221.37726521],
[-236.88574241, -223.46474078],
[-237.02510397, -221.70668965],
[-237.48219251, -222.95298654],
[-235.65481301, -221.80439823],
[-237.33740071, -223.28895162],
[-235.5875405 , -220.51907202],
[-236.60302944, -223.92095582],
[-237.70040704, -221.3117112 ],
[-236.63442124, -222.81507499],
[-236.49994294, -222.97590817],
[-236.46644445, -221.9596193 ],
[-236.77041741, -223.17919881],
[-236.90858264, -221.74010082],
[-236.48252201, -223.10869896],
[-235.66661422, -221.08546251],
[-238.20972126, -223.6992055 ],
[-236.20544399, -222.61327618],
[-236.7863424 , -222.45665124],
[-235.72488443, -219.48460968],
[-238.34759568, -225.18437751],
[-234.40492207, -220.52084571],
[-234.83867816, -220.21992665],
[-239.22293979, -223.84444469],
[-237.73050563, -224.23648546],
[-235.25756016, -221.83157323],
[-236.93119909, -222.43439465],
[-237.29959493, -222.38642482],
[-236.1937444 , -222.44029694],
[-237.41363895, -222.46347395],
[-236.53861231, -222.31390391],
[-235.89233955, -220.69546983],
[-236.37638819, -223.36620641],
[-238.02193286, -223.19325483],
[-234.81332517, -221.69511121],
[-238.20247509, -223.30282019],
[-234.99265702, -221.20956625],
[-237.29190838, -222.65033895],
[-236.25936623, -221.29385343],
[-238.07593087, -224.17730105],
[-235.99246078, -221.57970378],
[-237.25039025, -221.78730947],
[-237.15499927, -223.52751344],
[-236.25601466, -221.39604303],
[-236.42976242, -222.24011873],
[-234.83888338, -220.41922648],
[-237.43999014, -224.00850375],
[-236.19376763, -221.67434784],
[-237.48275285, -223.3269265 ],
[-237.65809153, -222.09979268],
[-236.0636132 , -223.33236215],
[-235.95739397, -221.03451575],
[-237.13027815, -223.35741516],
[-235.60765256, -220.65476655],
[-237.19489699, -223.64037441],
[-236.93995612, -221.95539417],
[-237.16138278, -222.61974611],
[-235.92250514, -222.00884552],
[-236.06371931, -222.26824449],
[-236.77220385, -222.08002567],
[-236.9167308 , -222.3903247 ],
[-238.21127984, -223.71716139],
[-236.02047971, -221.76679757],
[-236.4515211 , -222.4118829 ],
[-236.8934018 , -222.62108413],
[-236.36083692, -222.02374839],
[-236.51595133, -222.19656648],
[ 557.78030208, 23.48846147],
[ 557.78657655, 23.47740815],
[ 557.77409093, 23.49469241],
[ 557.81135778, 23.48303826],
[ 557.83902701, 23.51454495],
[ 557.69686789, 23.52229184],
[ 557.9752935 , 23.43570964],
[ 557.77416235, 23.53350662],
[ 557.77582972, 23.52275855],
[ 557.87823377, 23.61646299],
[ 557.48374309, 23.31502225],
[ 557.95343212, 23.36924087],
[ 557.99973447, 23.67798822],
[ 558.0419255 , 23.67688101],
[ 556.75206291, 23.08126355],
[ 558.52949843, 23.52749005],
[ 557.72727324, 23.58512427],
[ 557.45530446, 23.53998565],
[ 557.79860287, 23.41526684],
[ 557.29884812, 23.40112721],
[ 558.46764579, 23.36267189],
[ 557.76831041, 23.50560019],
[ 558.09110819, 23.83666 ],
[ 556.84467317, 22.7734904 ],
[ 559.30185281, 24.56261359],
[ 557.02563561, 22.95028173],
[ 557.92367001, 23.68065294],
[ 557.90691012, 23.86293983],
[ 557.04750596, 22.4349778 ],
[ 558.84522648, 23.86118714],
[ 556.73652124, 22.89229848],
[ 558.53611815, 24.1244896 ],
[ 557.42961709, 23.01037602],
[ 558.50746927, 24.65653399],
[ 557.18744829, 22.89126156],
[ 557.68970521, 23.25660619],
[ 557.38034071, 23.58963729],
[ 557.42144702, 23.59205635],
[ 558.3036823 , 23.8240342 ],
[ 559.12564093, 23.51308743],
[ 557.00487176, 23.27532945],
[ 556.75756869, 23.30305547],
[ 559.36921929, 23.39702826],
[ 556.81260191, 23.54846411],
[ 559.01500188, 24.00270626],
[ 557.05307911, 23.24237372],
[ 557.43566105, 23.79454668],
[ 558.09855535, 23.37032459],
[ 557.83881308, 22.88169276],
[ 554.56193189, 20.70638429],
[ 560.98557535, 26.4952269 ],
[ 559.4456611 , 24.2891323 ],
[ 557.59173649, 23.3545558 ],
[ 556.8526621 , 23.33745079],
[ 556.09322306, 22.24189727],
[ 556.98858203, 23.77742733],
[ 560.55921214, 23.6311708 ],
[ 556.87796349, 23.18954614],
[ 557.77010278, 23.98122176],
[ 559.21806576, 24.91316565],
[ 557.28543267, 22.20277088],
[ 556.85161756, 23.68616762],
[ 557.94058506, 24.32370988],
[ 557.89350618, 22.36375557],
[ 558.33504941, 23.22862785],
[ 556.39021429, 24.82277972],
[ 560.23431993, 22.41331061],
[ 556.43457577, 23.96325386],
[ 558.28179207, 22.78878159],
[ 553.48615604, 23.35912539],
[ 563.649538 , 24.75674352],
[ 555.22480614, 22.39595006],
[ 558.48750866, 24.92509113],
[ 557.33676336, 22.68786987],
[ 557.49297151, 22.9678329 ],
[ 559.04175301, 24.11914439],
[ 558.85272413, 23.4937694 ],
[ 555.41138959, 23.02127048],
[ 558.34456361, 23.55995815],
[ 557.57817056, 23.68153876],
[ 559.34725573, 24.42267963],
[ 557.26313468, 22.36462266],
[ 557.28170372, 24.46778913],
[ 558.13960043, 22.44575532],
[ 558.7205231 , 23.86212437],
[ 556.51097715, 22.93042715],
[ 557.63925883, 24.57190663],
[ 557.95864236, 22.83960219],
[ 557.41924431, 23.48070605],
[ 559.52250744, 24.0554022 ],
[ 556.11996466, 23.20959286],
[ 557.80568878, 23.46329163],
[ 557.19563963, 22.718299 ],
[ 558.79433352, 23.70264118],
[ 557.95478109, 24.20309529],
[ 558.81749612, 23.29839148],
[ 556.77854696, 23.03826987],
[ 556.85620994, 23.5875734 ],
[ 559.60307088, 23.88178437],
[ 557.60641847, 23.52754109],
[ 556.51303182, 23.30117411],
[ 558.97243169, 23.88038059],
[ 557.06759578, 23.25614134],
[ 557.57736831, 23.3151427 ],
[ 557.99326758, 23.41642631],
[ 557.69535063, 23.52055539],
[-330.49623678, 204.63177927],
[-330.48687831, 204.61748196],
[-330.47596154, 204.61709212],
[-330.56190426, 204.7288047 ],
[-330.52672442, 204.57380887],
[-330.43931999, 204.65189979],
[-330.60496276, 204.6959889 ],
[-330.42213563, 204.62693736],
[-330.40344234, 204.45782583],
[-330.72234249, 205.110622 ],
[-330.50651333, 204.52792838],
[-330.28895358, 204.38582412],
[-330.69857762, 204.8216316 ],
[-330.37129335, 204.61156846],
[-330.55355326, 204.66551459],
[-330.47257205, 204.4651604 ],
[-330.57180158, 204.95572727],
[-330.40915755, 204.53164132],
[-330.63913328, 204.25510348],
[-330.27873877, 205.10080122],
[-330.69329799, 204.67949606],
[-330.42104478, 204.44265984],
[-330.54349129, 204.31544708],
[-330.12411955, 204.29052866],
[-330.75325966, 205.4935008 ],
[-331.08219524, 204.60618508],
[-328.78443345, 203.86316298],
[-332.36202932, 205.70237444],
[-329.49198804, 203.54946633],
[-330.72019779, 205.5844651 ],
[-330.83746074, 204.40399927],
[-329.91778067, 203.91848653],
[-331.50085978, 206.1622208 ],
[-329.01669917, 202.59359178],
[-331.18032257, 206.40565512],
[-330.85747495, 203.8352504 ],
[-329.79647706, 205.19487337],
[-331.03367586, 204.4347803 ],
[-330.98896386, 205.04237199],
[-329.13218979, 201.92476687],
[-331.72768394, 206.82650452],
[-330.01526677, 204.72351573],
[-329.75728257, 203.75508104],
[-332.05499455, 206.52757798],
[-330.02865615, 203.18676144],
[-332.0325883 , 206.56627054],
[-329.43346802, 204.0015991 ],
[-331.02562204, 204.11839407],
[-329.11363624, 203.84427733],
[-330.52758022, 205.55396543],
[-330.93033161, 203.04216168],
[-330.68605732, 206.7567235 ],
[-328.42711673, 202.2776608 ],
[-330.56321919, 204.31764144],
[-333.08367704, 207.99735973],
[-330.65385763, 204.72275808],
[-330.09276197, 203.7049107 ],
[-329.95288941, 203.87804551],
[-330.79189571, 206.19320977],
[-330.41442723, 204.02019103],
[-330.19749321, 203.39195211],
[-327.35523341, 200.91253803],
[-334.00271683, 209.92360674],
[-328.91035046, 202.53091472],
[-331.30211814, 204.86140794],
[-329.73801651, 202.85856816],
[-331.38149616, 207.07678546],
[-332.13002486, 206.73783294],
[-328.86798638, 203.10503107],
[-331.58787388, 205.24605527],
[-329.60012227, 203.36930754],
[-330.86051542, 205.29514954],
[-328.37749944, 202.72687786],
[-331.87528771, 207.36582639],
[-330.62146462, 203.74846678],
[-331.45872728, 203.94568428],
[-330.04635104, 205.66623843],
[-330.99098212, 204.97916383],
[-329.85330721, 202.89960273],
[-331.153215 , 207.30135761],
[-329.83503928, 203.4128321 ],
[-330.319143 , 204.13247582],
[-330.22248161, 205.19244935],
[-330.32819763, 204.85952947],
[-331.5266198 , 204.29317565],
[-329.87724539, 204.30010139],
[-330.82446353, 204.42368733],
[-330.66104205, 205.93255728],
[-329.94202213, 202.74802265],
[-330.8497396 , 204.70332776],
[-331.38117512, 206.20314629],
[-329.85586754, 203.58494602],
[-330.59775054, 204.26071712],
[-329.94811216, 204.8314185 ],
[-331.49949484, 205.61226459],
[-330.2655716 , 204.82347597],
[-330.37857427, 204.31801088],
[-330.61075079, 204.9873505 ],
[-330.76721124, 204.92169998],
[-330.04385968, 203.68423393],
[-330.4280387 , 204.68944121],
[-330.57702698, 204.80718571],
[-330.25057119, 204.35809962]])
In [39]:
plt.scatter(transf[:,0], transf[:,1], c=y.values.ravel())
Out[39]:
<matplotlib.collections.PathCollection at 0x121999220>
In [45]:
X.iloc[5,:].values
Out[45]:
array([ 0.40247615, 0. , 62.2774886 , ..., 0. ,
0. , 0. ])
In [48]:
lda.predict([X.iloc[200,:].values])
Out[48]:
array([1])
In [79]:
mdl = MiniSom(12, 12, X.values.shape[1])
mdl.random_weights_init(X.values)
mdl.train_random(X.values, 100)
In [80]:
quantization = mdl.quantization(X.values)
# for i in range(X.values.shape[0]):
# x = X.values[i]
# winner = mdl.winner(x)
# print('{} - {} - {}'.format(y.values.ravel()[i], winner, quantization[i]))
plt.pcolor(mdl.distance_map().T)
plt.show()
In [76]:
agl = AgglomerativeClustering(n_clusters=3)
agl.fit(X)
Out[76]:
AgglomerativeClustering(n_clusters=3)
In [77]:
silhouette_score(X, agl.labels_)
Out[77]:
0.8461693414879796
In [78]:
homogeneity_score(y.values.ravel(), agl.labels_)
Out[78]:
1.0
In [72]:
linked = linkage(X, 'ward')
labelList = agl.labels_
plt.figure(figsize=(10, 7))
dendrogram(linked,
orientation='top',
labels=labelList,
distance_sort='descending',
show_leaf_counts=True)
plt.show()
