通过对两个字典进行操作创建 pandas 数据框

Question

我正在尝试创建一个 pandas 数据框，其中 row 来自一个字典， column 来自另一个字典，并且定义了 row[i]-column[j] 的值通过对两个字典的键值对进行的任何操作（即 row[dict1[key]]-column[dict2[key]] 的值可以从接受值的函数计算：dict1[key] 和 dict2[key]）。

到目前为止，我的代码如下所示：

# -*- coding: utf-8 -*-
import sys
import os
import pandas as pd
from optparse import OptionParser
from sklearn.preprocessing import MinMaxScaler
from joblib import Parallel, delayed
import pybedtools
from subprocess import call
from collections import defaultdict
import numpy as np
from skbio.sequence import DNA
from skbio.alignment import local_pairwise_align_ssw
class sequenceCompare:

class sequenceCompare:

    '''Common class for comparing multifasta files'''

    def __init__(
        self,
        fasta1,
        fasta2
        ):
        self.fasta1 = fasta1
        self.fasta2 = fasta2

    def computeScore(self):
        sequenceList1 = {}
        sequenceList2 = {}
        score_matrix = pd.DataFrame([])
        with open(self.fasta1) as file_one:
            sequenceList1 = {line.strip(">\n"):next(file_one).rstrip() for line in file_one}        
        with open(self.fasta2) as file_two:
            sequenceList2 = {line.strip(">\n"):next(file_two).rstrip() for line in file_two} 
        #Is there any way to make following step parallel 
        for key1, value1 in sequenceList1.items():
            for key2, value2 in sequenceList2.items():
                    alignment, score, start_end_positions = local_pairwise_align_ssw(DNA(value1), DNA(value2))
                    #Store value of score in dataframe column key1 and row key2

例如。

Sequence list 1: 
>A1
AAACCTTGGG
>A2
CCCAAAATTT
>A3
CCTTAAGGG

Sequence list 2:
>B1
GGTTAACC
>B2
GATCATCCA
>B3
CCAAAATTC

对两个字典进行操作后得到的数据帧应该是这样的：

Dataframe: 
       A1          A2          A3
B1 dist(A1,B1) dist(A2,B1) dist(A3,B1)
B2 dist(A1,B2) dist(A2,B2) dist(A3,B2)
B3 dist(A1,B3) dist(A2,B3) dist(A3,B3)

执行此操作最有效（并希望并行）的方法是什么？

Answer 1

这段代码可以解决问题：

dict1 = {'B2': 'GATCATCCA', 'B3': 'CCAAAATTC', 'B1': 'GGTTAACC'}
dict2 = {'A2': 'CCCAAAATTT', 'A3': 'CCTTAAGGG', 'A1': 'AAACCTTGGG'}
finaldict = {}
for key1, value1 in dict1.items():
    for key2, value2 in dict2.items():
        #--> apply func taking (value2, value1) as input <..#
        try:
            finaldict[key2].update({key1:func})
        except:
            finaldict[key2] = {key1:func}
pd.DataFrame(finaldict)

Answer 2

查看文档，似乎构建一个 StripedSmithWaterman object and use it multiple times instead of using local_pairwise_align_ssw every time. However, it does not seem to provide parallelism on its own (which is strange, because the library on which it is based claims to implement SIMD parallelism, so I may be wrong), but you can use regular Python multiprocessing 来并行化事物更有效：

# -*- coding: utf-8 -*-
import sys
import os
import pandas as pd
from optparse import OptionParser
from sklearn.preprocessing import MinMaxScaler
from joblib import Parallel, delayed
import pybedtools
from subprocess import call
from multiprocessing import Pool
from itertools import repeat
from collections import defaultdict
import numpy as np
from skbio.sequence import DNA
from skbio.alignment import StripedSmithWaterman


def compute_scores(dna1, dnas2):
    # StripedSmithWaterman docs:
    # http://scikit-bio.org/docs/0.4.2/generated/skbio.alignment.StripedSmithWaterman.html
    ssw1 = StripedSmithWaterman(dna1)
    # AlignmentStructure docs:
    # http://scikit-bio.org/docs/0.4.2/generated/skbio.alignment.AlignmentStructure.html
    return [ssw1(dna2).optimal_alignment_score for dna2 in dnas2]

class sequenceCompare:

    '''Common class for comparing multifasta files'''

    def __init__(
        self,
        fasta1,
        fasta2
        ):
        self.fasta1 = fasta1
        self.fasta2 = fasta2

    def computeScore(self):
        sequenceList1 = {}
        sequenceList2 = {}
        score_matrix = pd.DataFrame([])
        with open(self.fasta1) as file_one:
            sequenceList1 = {line.strip(">\n"):next(file_one).rstrip() for line in file_one}
        with open(self.fasta2) as file_two:
            sequenceList2 = {line.strip(">\n"):next(file_two).rstrip() for line in file_two}
        with Pool(os.cpu_count()) as p:
            values2 = list(sequenceList2.values())
            data = p.starmap(compute_scores, zip(sequenceList1.values(), repeat(values2)))
            df = pd.DataFrame(data, columns=list(sequenceList1.keys()), index=list(sequenceList2.keys()))
            # df contains the resulting data frame

通过对两个字典进行操作创建 pandas 数据框

Create pandas dataframe by operating on two dictionaries

python

dictionary

pandas

joblib