Getting error sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping" in postgres insert
Getting error sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping" in postgres insert
我已经声明了我的连接函数如下..
def create_psql_engine(db):
#conn_string = "postgresql+psycopg2://myuser:******@someserver:port/somedb"
conn_string = "postgresql+psycopg2://%s:%s@%s:%s/%s" % (
db.get('user'),
db.get('password'),
db.get('host'),
db.get('port'),
db.get('dbname')
)
conn_args = {
"sslmode": db.get('sslmode'),
"sslcompression": db.get('sslcompression'),
"sslrootcert": db.get('sslrootcert'),
"sslcert": db.get('sslcert'),
"sslkey": db.get('sslkey'),
"pool_pre_ping": True,
"pool_recycle": 300
}
try:
engine = create_engine(conn_string, connect_args=conn_args)
logging.info("sqlalchemy Engine to database created using psycopg2... ")
return engine
except psycopg2.Error as error:
logging.error(error)
在我的程序中，我连接如下。
# Create PostgreSQL Engine
self.engine = abc.create_psql_engine(dbParam)
print('PSQL Engine Created to Table : {}'.format(self.tableName))
然后我使用 pandas to_sql 插入到 DB
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
在没有以下 2 个选项的情况下，它一直在工作。
"pool_pre_ping": True,
"pool_recycle": 300
但是在处理大容量时我开始连接中断。这就是我尝试使用这两个参数的解决方案的原因。
但是，当我尝试时出现此错误。任何线索我做错了什么。
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
完整的错误日志如下..
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
psycopg2.ProgrammingError: invalid dsn: invalid connection option "pool_pre_ping"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/apps/python/loadtoPostgreSQL.py", line 275, in <module>
MX123D.processFiles(jobIdentifier)
File "/opt/apps/python/loadtoPostgreSQL.py", line 246, in processFiles
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
File "/usr/local/lib64/python3.6/site-packages/pandas/core/generic.py", line 2615, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 598, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1393, in to_sql
table.create()
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 721, in create
if self.exists():
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 708, in exists
return self.pd_sql.has_table(self.name, self.schema)
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1431, in has_table
self.connectable.dialect.has_table, name, schema or self.meta.schema
File "<string>", line 2, in run_callable
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/deprecations.py", line 390, in warned
return fn(*args, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3074, in run_callable
with self.connect() as conn:
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3166, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3216, in _wrap_pool_connect
e, dialect, self
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 2070, in _handle_dbapi_exception_noconnection
sqlalchemy_exception, with_traceback=exc_infoÝ2¨, from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
(Background on this error at: https://sqlalche.me/e/14/f405)
我自己在这里找到了 issue.Sharing 以便它可以帮助一些人。
问题是参数在错误的地方给出:-)
应该如下给出。
engine = create_engine(conn_string, connect_args=conn_args,pool_pre_ping=True,pool_recycle=300)
我已经声明了我的连接函数如下..
def create_psql_engine(db):
#conn_string = "postgresql+psycopg2://myuser:******@someserver:port/somedb"
conn_string = "postgresql+psycopg2://%s:%s@%s:%s/%s" % (
db.get('user'),
db.get('password'),
db.get('host'),
db.get('port'),
db.get('dbname')
)
conn_args = {
"sslmode": db.get('sslmode'),
"sslcompression": db.get('sslcompression'),
"sslrootcert": db.get('sslrootcert'),
"sslcert": db.get('sslcert'),
"sslkey": db.get('sslkey'),
"pool_pre_ping": True,
"pool_recycle": 300
}
try:
engine = create_engine(conn_string, connect_args=conn_args)
logging.info("sqlalchemy Engine to database created using psycopg2... ")
return engine
except psycopg2.Error as error:
logging.error(error)
在我的程序中，我连接如下。
# Create PostgreSQL Engine
self.engine = abc.create_psql_engine(dbParam)
print('PSQL Engine Created to Table : {}'.format(self.tableName))
然后我使用 pandas to_sql 插入到 DB
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
在没有以下 2 个选项的情况下，它一直在工作。
"pool_pre_ping": True,
"pool_recycle": 300
但是在处理大容量时我开始连接中断。这就是我尝试使用这两个参数的解决方案的原因。
但是，当我尝试时出现此错误。任何线索我做错了什么。
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
完整的错误日志如下..
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
psycopg2.ProgrammingError: invalid dsn: invalid connection option "pool_pre_ping"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/apps/python/loadtoPostgreSQL.py", line 275, in <module>
MX123D.processFiles(jobIdentifier)
File "/opt/apps/python/loadtoPostgreSQL.py", line 246, in processFiles
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
File "/usr/local/lib64/python3.6/site-packages/pandas/core/generic.py", line 2615, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 598, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1393, in to_sql
table.create()
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 721, in create
if self.exists():
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 708, in exists
return self.pd_sql.has_table(self.name, self.schema)
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1431, in has_table
self.connectable.dialect.has_table, name, schema or self.meta.schema
File "<string>", line 2, in run_callable
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/deprecations.py", line 390, in warned
return fn(*args, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3074, in run_callable
with self.connect() as conn:
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3166, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3216, in _wrap_pool_connect
e, dialect, self
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 2070, in _handle_dbapi_exception_noconnection
sqlalchemy_exception, with_traceback=exc_infoÝ2¨, from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
(Background on this error at: https://sqlalche.me/e/14/f405)
我自己在这里找到了 issue.Sharing 以便它可以帮助一些人。 问题是参数在错误的地方给出:-) 应该如下给出。
engine = create_engine(conn_string, connect_args=conn_args,pool_pre_ping=True,pool_recycle=300)