是否可以从 Google App Engine 中一起获取多个对象？

Question

我们在 Google App Engine 中有大约 5,000 个 class 域对象，我们希望将域列表导出为 CSV。每个域都链接到 class DomainStateData:

的一个对象

class DomainStateData(db.Expando, ExpandoEntity):
    plan = db.ReferenceProperty(Plan)
    plan_expiration = db.DateTimeProperty()
    trial_expiration = db.DateTimeProperty()

    date_created = db.DateTimeProperty(auto_now_add=True, indexed=True)
    last_modified = db.DateTimeProperty(auto_now=True)


class Domain(db.Expando, ExpandoEntity, SocialIconsEntity):
    """
    Domain Model
    """
    domain = db.StringProperty(required=True)
    ...
    _state_data = db.ReferenceProperty(DomainStateData)

    @property
    def state_data(self):
        try:
            if not self._state_data:
                # try to get it, if not, build it
                sd = DomainStateData.get_by_key_name(self.key().name())
                if not sd:
                    sd = DomainStateData(key_name=self.key().name()).put()
                self._state_data = sd
                self.put()
                return self._state_data
            else:
                return self._state_data
        except ReferencePropertyResolveError:
            self._state_data = DomainStateData(key_name=self.key().name()).put()
            self.put()
            return self._state_data

我编写了一个将 100 个域导出到 CSV 的代码（需要 5 秒），但是如果我尝试获取所有 5,000 个域，我会超时，即 60 秒。是否可以在没有超时的情况下一起获取所有 DomainStateData 对象？这是我将域导出到 CSV 的代码：

import sys
sys.path.insert(0, 'libs')

import webapp2
import datetime
import csv
from models import Domain


class ExportAllDomainsToCsvHandler(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/csv'
        self.response.headers['Content-Disposition'] = 'attachment; filename="All Domains [{0}].csv"'.format(str(datetime.date.today()))
        writer = csv.writer(self.response.out)
        writer.writerow(["Domain", "Current state", "Plan expiration date", "Trial expiration date", "Current oauth user"])
        all_domains = Domain.all().fetch(100)
        all_domains.sort(key=lambda domain: (0 if domain.state_data.plan_expiration is None else 1, domain.state_data.plan_expiration, 0 if domain.state_data.trial_expiration is None else 1, domain.state_data.trial_expiration, domain.domain))
        for domain in all_domains:
            if (domain.state_data.plan_expiration is None):
                domain_plan_expiration = "No plan expiration date"
            else:
                domain_plan_expiration = domain.state_data.plan_expiration.strftime('%Y-%m-%d')
            if (domain.state_data.trial_expiration is None):
                domain_trial_expiration = "No trial expiration date"
            else:
                domain_trial_expiration = domain.state_data.trial_expiration.strftime('%Y-%m-%d')
            writer.writerow([domain.domain, domain.cur_state.name, domain_plan_expiration, domain_trial_expiration, domain.admin])


app = webapp2.WSGIApplication([
    ("/csv/export_all_domains_to_csv", ExportAllDomainsToCsvHandler)
], debug=True)

Answer 1

好的，我找到了解决办法。我直接从数据库中获取所有 DomainStateData 对象，现在需要 35 秒才能创建包含所有域的 CSV。这是我的代码，我没有更改模型：

import sys
sys.path.insert(0, 'libs')

import webapp2
import datetime
import csv
from models import DomainStateData, Domain


class ExportAllDomainsToCsvHandler(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/csv'
        self.response.headers['Content-Disposition'] = 'attachment; filename="All Domains [{0}].csv"'.format(str(datetime.date.today()))
        writer = csv.writer(self.response.out)
        writer.writerow(["Domain", "Current state", "Plan expiration date", "Trial expiration date", "Current oauth user"])
        all_domain_state_data_dict = dict()
        all_domain_state_data = DomainStateData.all().fetch(1000000)
        all_domains = Domain.all().fetch(1000000)
        for domain_state_data in all_domain_state_data:
            all_domain_state_data_dict[domain_state_data.key().name()] = domain_state_data
        for domain in all_domains:
            if (domain.key().name() in all_domain_state_data_dict):
                domain.__state_data = all_domain_state_data_dict[domain.key().name()]
        all_domains.sort(key=lambda domain: (0 if domain.__state_data.plan_expiration is None else 1, domain.__state_data.plan_expiration, 0 if domain.__state_data.trial_expiration is None else 1, domain.__state_data.trial_expiration, domain.domain))
        for domain in all_domains:
            if (domain.__state_data.plan_expiration is None):
                domain_plan_expiration = "No plan expiration date"
            else:
                domain_plan_expiration = domain.__state_data.plan_expiration.strftime('%Y-%m-%d')
            if (domain.__state_data.trial_expiration is None):
                domain_trial_expiration = "No trial expiration date"
            else:
                domain_trial_expiration = domain.__state_data.trial_expiration.strftime('%Y-%m-%d')
            writer.writerow([domain.domain, domain.cur_state.name, domain_plan_expiration, domain_trial_expiration, domain.admin])

app = webapp2.WSGIApplication([
    ("/csv/export_all_domains_to_csv", ExportAllDomainsToCsvHandler)
], debug=True)

是否可以从 Google App Engine 中一起获取多个对象？

Is it possible to fetch many objects together from Google App Engine?

python

csv

google-app-engine