Push

Dockerfile

@@ -9,6 +9,7 @@ RUN useradd -m -u 1000 user
 USER user
 ENV HOME=/home/user \
 	PATH=/home/user/.local/bin:$PATH
+
 # Set the working directory in the container.
 WORKDIR $HOME/app
 

lilac/__init__.py

@@ -8,6 +8,8 @@ from .data_loader import create_dataset
 from .db_manager import get_dataset, set_default_dataset_cls
 from .embeddings import *  # noqa: F403
 from .embeddings.default_vector_stores import register_default_vector_stores
+from .env import *  # noqa: F403
+from .env import LilacEnvironment
 from .schema import *  # noqa: F403
 from .server import start_server, stop_server
 from .signals import *  # noqa: F403
@@ -39,4 +41,5 @@ __all__ = [
   'EmbeddingConfig',
   'SignalConfig',
   'DatasetSettings',
+  'LilacEnvironment',
 ]

lilac/env.py

@@ -1,25 +1,64 @@
 """Load environment variables from .env file."""
 import os
-from typing import Any, Literal, Optional, Union, cast
+from typing import Any, Optional, cast
 
 from dotenv import load_dotenv
+from pydantic import BaseModel
+from pydantic import Field as PydanticField
 
-EnvironmentKeys = Union[Literal['LILAC_DATA_PATH'],
-                        # Authentication on the demo.
-                        Literal['LILAC_AUTH_ENABLED'], Literal['GOOGLE_CLIENT_ID'],
-                        Literal['GOOGLE_CLIENT_SECRET'], Literal['LILAC_OAUTH_SECRET_KEY'],
-                        # DuckDB accessing GCS.
-                        Literal['GCS_REGION'], Literal['GCS_ACCESS_KEY'], Literal['GCS_SECRET_KEY'],
-                        # Embedding API keys.
-                        Literal['OPENAI_API_KEY'], Literal['COHERE_API_KEY'],
-                        Literal['PALM_API_KEY'],
-                        # HuggingFace demos.
-                        Literal['HF_USERNAME'], Literal['HF_STAGING_DEMO_REPO'],
-                        Literal['SPACE_ID'], Literal['HF_ACCESS_TOKEN'],
-                        # DuckDB
-                        Literal['DUCKDB_USE_VIEWS'],
-                        # Debugging
-                        Literal['DEBUG'], Literal['DISABLE_LOGS']]
+
+# NOTE: This is created for documentation, but isn't parsed by pydantic until we update to 2.0.
+class LilacEnvironment(BaseModel):
+  """Lilac environment variables.
+
+  These can be set with operating system environment variables to override behavior.
+
+  For python, see: https://docs.python.org/3/library/os.html#os.environ
+
+  For bash, see: https://www.gnu.org/software/bash/manual/bash.html#Environment
+  """
+  # General Lilac environment variables.
+  LILAC_DATA_PATH: str = PydanticField(
+    description='The Lilac data path where datasets, concepts, caches are stored.',)
+  DEBUG: str = PydanticField(
+    description='Turn on Lilac debug mode to log queries and timing information.')
+  DISABLE_LOGS: str = PydanticField(description='Disable log() statements to the console.')
+
+  # API Keys.
+  OPENAI_API_KEY: str = PydanticField(
+    description='The OpenAI API key, used for computing `openai` embeddings and generating positive '
+    'examples for concept seeding.')
+  COHERE_API_KEY: str = PydanticField(
+    description='The Cohere API key, used for computing `cohere` embeddings.')
+  PALM_API_KEY: str = PydanticField(
+    description='The PaLM API key, used for computing `palm` embeddings.')
+
+  # HuggingFace demo.
+  HF_ACCESS_TOKEN: str = PydanticField(
+    description='The HuggingFace access token, used for downloading data to a space from a '
+    'private dataset. This is also required if the HuggingFace space is private.')
+
+  # DuckDB.
+  DUCKDB_USE_VIEWS: str = PydanticField(
+    description='Whether DuckDB uses views (1), or DuckDB tables (0). Views allow for much less '
+    'RAM consumption, with a runtime query penalty. When using DuckDB tables (0), demos will '
+    'take more RAM but be much faster during query time.')
+
+  # Authentication.
+  LILAC_AUTH_ENABLED: str = PydanticField(
+    description='Set to true to enable read-only mode, disabling the ability to add datasets & '
+    'compute dataset signals. When enabled, `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` and '
+    '`LILAC_OAUTH_SECRET_KEY` should also be set.')
+  GOOGLE_CLIENT_ID: str = PydanticField(
+    description=
+    'The Google OAuth client ID. Required when `LILAC_AUTH_ENABLED=true`. Details can be found at '
+    'https://developers.google.com/identity/protocols/oauth2.')
+  GOOGLE_CLIENT_SECRET: str = PydanticField(
+    description='The Google OAuth client secret. Details can be found at '
+    'https://developers.google.com/identity/protocols/oauth2.')
+  LILAC_OAUTH_SECRET_KEY: str = PydanticField(
+    description='The Google OAuth random secret key. Details can be found at '
+    'https://developers.google.com/identity/protocols/oauth2.')
 
 
 def _init_env() -> None:
@@ -50,7 +89,7 @@ def _init_env() -> None:
       raise ValueError('Missing `LILAC_OAUTH_SECRET_KEY` when `LILAC_AUTH_ENABLED=true`')
 
 
-def env(key: EnvironmentKeys, default: Optional[Any] = None) -> Any:
+def env(key: str, default: Optional[Any] = None) -> Any:
   """Return the value of an environment variable."""
   return os.environ.get(key, default)