Merge branch 'main' of https://github.com/sdss/sdssdb

Author: Michael Blanton

.github/workflows/pythonapp.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/release.yml

@@ -23,12 +23,12 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Create release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           name: sdssdb ${{ github.ref_name }}
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

CHANGELOG.rst

@@ -5,6 +5,35 @@ Changelog
 
 This document records the main changes to the ``sdssdb`` code.
 
+* Add method ``get_database_uri()`` to ``DatabaseConnection``.
+
+* :release:`0.12.0 <2024-06-28>`
+* Updated ``sdss_id_to_catalog`` view models to use double underscore separating table name from primary key.
+* Very significantly speed up the reflection of the database by caching the schema metadata.
+* Add ``targetdb`` ``targeting_generation`` and ``targeting_generation_to_target`` tables and models.
+* Allow passing additional connection parameters to ``set_profile()`` which will override the profile defaults.
+
+* :release:`0.11.4 <2024-04-30>`
+* Add columns ``too_metadata.last_modified_date``, ``too_target.observe_from_mjd``, and ``too_target.added_date``.
+* Rename column ``too_target.expiration_date`` to ``too_target.observe_until_mjd``.
+
+* :release:`0.11.3 <2024-04-28>`
+* Add columns ``fiber_type`` and ``assigned`` to ``opsdb.assignment_to_focal``.
+
+* :release:`0.11.2 <2024-04-24>`
+* Add column ``can_offset`` to ``too_metadata``.
+
+* :release:`0.11.1 <2024-04-24>`
+* Fix typo in `CatalogToToO_Target` model name.
+
+* :release:`0.11.0 <2024-04-24>`
+* Explicitly define ``CatalogToXXX`` models instead of loading them dynamically.
+
+* :release:`0.10.0 <2024-04-23>`
+* Add ``catalogdb.too_target`` and ``catalogdb.too_metadata`` tables and models.
+* Add new cadences.
+* Add ``disabled`` flag to tile.
+
 * :release:`0.9.0 <2024-03-08>`
 * Add new schema for ``astra`` to ``sdss5db``.
 * Added new peewee and sqlalchemy ORMs for ``astra`` schema.

python/sdssdb/connection.py

@@ -6,30 +6,29 @@
 # @Filename: database.py
 # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause)
 
+from __future__ import annotations
+
 import abc
 import importlib
 import os
 import re
 import socket
 
+import peewee
 import pgpasslib
 import six
-
+from peewee import OperationalError, PostgresqlDatabase
+from playhouse.postgres_ext import ArrayField
+from playhouse.reflection import Introspector, UnknownField
 from sqlalchemy import MetaData, create_engine
 from sqlalchemy.engine import url
 from sqlalchemy.exc import OperationalError as OpError
 from sqlalchemy.orm import scoped_session, sessionmaker
 
-import peewee
-from peewee import OperationalError, PostgresqlDatabase
-from playhouse.postgres_ext import ArrayField
-from playhouse.reflection import Introspector, UnknownField
-
 import sdssdb
 from sdssdb import config, log
 from sdssdb.utils.internals import get_database_columns
 
-
 __all__ = ['DatabaseConnection', 'PeeweeDatabaseConnection', 'SQLADatabaseConnection']
 
 
@@ -44,6 +43,30 @@ def _should_autoconnect():
         return sdssdb.autoconnect
 
 
+def get_database_uri(
+    dbname: str,
+    host: str | None = None,
+    port: int | None = None,
+    user: str | None = None,
+    password: str | None = None,
+):
+    """Returns the URI to the database."""
+
+    if user is None and password is None:
+        auth: str = ""
+    elif password is None:
+        auth: str = f"{user}@"
+    else:
+        auth: str = f"{user}:{password}@"
+
+    host_port: str = f"{host or ''}" if port is None else f"{host or ''}:{port}"
+
+    if auth == "" and host_port == "":
+        return f"postgresql://{dbname}"
+
+    return f"postgresql://{auth}{host_port}/{dbname}"
+
+
 class DatabaseConnection(six.with_metaclass(abc.ABCMeta)):
     """A PostgreSQL database connection with profile and autoconnect features.
 
@@ -109,7 +132,7 @@ def __repr__(self):
         return '<{} (dbname={!r}, profile={!r}, connected={})>'.format(
             self.__class__.__name__, self.dbname, self.profile, self.connected)
 
-    def set_profile(self, profile=None, connect=True):
+    def set_profile(self, profile=None, connect=True, **params):
         """Sets the profile from the configuration file.
 
         Parameters
@@ -119,6 +142,9 @@ def set_profile(self, profile=None, connect=True):
             determine the profile.
         connect : bool
             If True, tries to connect to the database using the new profile.
+        params
+            Connection parameters (``user``, ``host``, ``port``, ``password``)
+            that will override the profile values.
 
         Returns
         -------
@@ -157,6 +183,8 @@ def set_profile(self, profile=None, connect=True):
                             self._config['host'] = None
                         break
 
+        self._config.update(params)
+
         if connect:
             if self.connected and self.profile == previous_profile:
                 pass
@@ -276,7 +304,16 @@ def list_profiles(profile=None):
 
         return config[profile]
 
-    @abc.abstractproperty
+    def get_connection_uri(self):
+        """Returns the URI to the database connection."""
+
+        params = self.connection_params
+        if not self.connected or params is None:
+            raise RuntimeError('The database is not connected.')
+
+        return get_database_uri(self.dbname, **params)
+
+    @abc.abstractmethod
     def connection_params(self):
         """Returns a dictionary with the connection parameters.
 

python/sdssdb/etc/sdssdb.yml

@@ -90,3 +90,7 @@ tunnel_operations:
     admin: sdss
     host: localhost
     port: 7502
+
+tunnel_pipelines:
+    host: localhost
+    port: 7602

python/sdssdb/peewee/__init__.py

@@ -148,17 +148,14 @@ def reflect(self):
 
         # Lists tables in the schema. This is a bit of a hack but
         # faster than using database.table_exists because it's cached.
-        database.get_fields(table_name, schema)
-        schema_tables = database._metadata[schema].keys()
+        metadata = database._metadata
+        if schema not in metadata or len(metadata[schema]) == 0:
+            database.get_fields(table_name, schema, cache=False)
+
+        schema_tables = metadata[schema].keys()
 
         if table_name not in schema_tables:
-            # Give it another try without caching. This is sometimes necessary
-            # for tables things like catalog_to_X table that are created
-            # dynamically.
-            database.get_fields(table_name, schema, cache=False)
-            schema_tables = database._metadata[schema].keys()
-            if table_name not in schema_tables:
-                return
+            return
 
         for index in meta.indexes:
             if hasattr(index, 'reflected') and index.reflected:

python/sdssdb/peewee/sdss5db/astradb.py

@@ -33,7 +33,6 @@ class Source(AstraBase):
     gaia_dr3_source_id = BigIntegerField(null=True)
     tic_v8_id = BigIntegerField(null=True)
     healpix = IntegerField(null=True)
-    carton_0 = TextField()
     lead = TextField(null=True)
     version_id = IntegerField(null=True)
     catalogid = BigIntegerField(null=True)