Add column 'retrieved_by' to web_page_snapshot

opdb/db/db_test.py

@@ -34,6 +34,7 @@ def test_add_web_page_snapshot(opdb_db: Db):
         snapshot_date=datetime.datetime.now(tz=datetime.timezone.utc),
         snapshot_url=None,
         retrieved_at=date,
+        retrieved_by="localhost",
         response_headers={"Content-Length": "7"},
         content=b"foo bar",
     )

opdb/db/models.py

@@ -44,6 +44,10 @@ class WebPageSnapshot(_BaseModel):
     """Moment the snapshot was downloaded by opdb and inserted in the DB (differs from
     :attr:`snapshot_date` if the snapshot was taken by a proxy)."""
 
+    retrieved_by: str
+    """Unique string identifying the worker that downloaded the snapshot."""
+    # TODO: define its format
+
     response_headers: dict[str, str]
     """Response headers of the webpage"""
 

opdb/db/models_test.py

@@ -34,6 +34,7 @@ def test_naive_datetime():
             snapshot_date=tz_date,
             snapshot_url=None,
             retrieved_at=naive_date,
+            retrieved_by="localhost",
             response_headers={"Content-Length": b"7"},
             content=b"foo bar",
         )
@@ -44,6 +45,7 @@ def test_naive_datetime():
             snapshot_date=naive_date,
             snapshot_url=None,
             retrieved_at=tz_date,
+            retrieved_by="localhost",
             response_headers={"Content-Length": b"7"},
             content=b"foo bar",
         )

opdb/db/orm_test.py

@@ -30,6 +30,7 @@ def test_db_schema():
             snapshot_date timestamptz NOT NULL,
             snapshot_url text,
             retrieved_at timestamptz NOT NULL,
+            retrieved_by text NOT NULL,
             response_headers jsonb NOT NULL,
             content bytea NOT NULL
         );

pyproject.toml

@@ -31,6 +31,7 @@ disable = [
     "no-else-return",
     "too-many-instance-attributes",
     "invalid-name",
+    "fixme",
     # mypy does it better:
     "no-member",
     "import-error",