Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ wikibots enriches Wikimedia Commons files with structured data (SDC) pulled from
- `src/wikibots/lib/claims.py` — `ClaimsMixin` with all `create_*` claim methods and hook stubs (`hook_creator_claim`, `hook_creator_target`, `hook_depicts_claim`, `hook_source_claim`).
- `src/wikibots/lib/bot.py` — `BaseBot(ClaimsMixin)`. Handles OAuth2 auth, Redis caching, HTTP sessions, file metadata, Commons/Wikidata API calls, and the main run loop.
- `src/wikibots/lib/wikidata.py` — Named constants for all Wikidata property (P-numbers) and entity (Q-numbers) IDs used across bots.
- `src/wikibots/flickr.py`, `inaturalist.py`, `pas.py`, `youtube.py` — Each bot overrides `treat_page()` and implements hooks for service-specific claim qualifiers.
- `src/wikibots/flickr.py`, `inaturalist.py`, `pas.py`, `youtube.py` — SDC bots: override `treat_page()` and implement hooks for service-specific claim qualifiers. Use `save()` to write claims via `wbeditentity`.
- `src/wikibots/flickrreview.py` — Wikitext bot: adds `{{Flickrreview}}` to Flickypedia uploads. Uses the `edit` API action directly (not `wbeditentity`). Override `skip_page()` for pre-loop filtering with Redis caching.

### Data flow

Expand Down Expand Up @@ -73,6 +74,7 @@ The `status` parameter determines the review outcome. Only `pass` means the lice

### Constraints

- Commons API with `formatversion: 2` returns `pages` as a **list**, not a dict keyed by pageid. Use `pages[0]` — not `pages.get(pageid)`.
- `redis` must stay on `<6.0.0` — Toolforge runs Redis server 6.0.16, and redis-py 7.x requires Redis 7.2+.
- `flickr_api` v3 (current) uses `PermissionDenied` for private/inaccessible photos — `PhotoIsPrivate` no longer exists.
- pywikibot config is accessed via `pwb.config` where `pwb` is imported from `pywikibot.scripts.wrapper`. Importing through the wrapper triggers proper pywikibot initialization — using `pywikibot.config` directly does not work. The ty `possibly-missing-submodule` warnings on `pwb.config` are expected (exit 0).
1 change: 1 addition & 0 deletions Procfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
flickr: python -m wikibots.flickr
flickrreview: python -m wikibots.flickrreview
inaturalist: python -m wikibots.inaturalist
pas: python -m wikibots.pas
youtube: python -m wikibots.youtube
84 changes: 42 additions & 42 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 7 additions & 6 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,23 @@ authors = [
readme = "README.md"
requires-python = ">=3.13,<4.0"
dependencies = [
"python-dateutil (>=2.9.0.post0,<3.0.0)",
"deepdiff (>=9.0.0,<10.0.0)",
"requests-oauthlib (>=2.0.0,<3.0.0)",
"google-api-python-client (>=2.166.0,<3.0.0)",
"flickr-photos-api (>=3.0.0,<4.0.0)",
"flickr-url-parser (>=1.11.0,<2.0.0)",
"redis (>=5.2.1,<6.0.0)",
"mwparserfromhell (>=0.7.2,<1.0.0)",
"python-dateutil (>=2.9.0.post0,<3.0.0)",
"redis (>=5.2.1,<6.0.0)",
"requests (>=2.32.0,<3.0.0)",
"requests-oauthlib (>=2.0.0,<3.0.0)",
]

[tool.poetry]
packages = [{include = "wikibots", from = "src"}]

[project.scripts]
flickr = "wikibots.flickr:main"
flickrreview = "wikibots.flickrreview:main"
inaturalist = "wikibots.inaturalist:main"
pas = "wikibots.pas:main"
youtube = "wikibots.youtube:main"
Expand All @@ -43,12 +44,12 @@ exclude = [

[dependency-groups]
dev = [
"ruff (>=0.15.9,<0.16.0)",
"ty (>=0.0.28,<0.0.29)",
"isort (>=8.0.1,<9.0.0)",
"pytest (>=9.0.3,<10.0.0)",
"pytest-mock (>=3.15.1,<4.0.0)",
"pytest-timeout (>=2.4.0,<3.0.0)"
"pytest-timeout (>=2.4.0,<3.0.0)",
"ruff (>=0.15.10,<0.16.0)",
"ty (>=0.0.31,<0.0.32)",
]


Expand Down
12 changes: 9 additions & 3 deletions src/wikibots/flickr.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,9 @@ def get_flickr_photo(self, flickr_photo_id: str) -> None:
while True:
try:
start = perf_counter()
self.photo = self.flickr_api.get_single_photo_info(photo_id=flickr_photo_id)
self.photo = self.flickr_api.get_single_photo_info(
photo_id=flickr_photo_id
)
logger.info(
f"Retrieved Flickr photo in {(perf_counter() - start) * 1000:.0f} ms"
)
Expand All @@ -159,9 +161,13 @@ def get_flickr_photo(self, flickr_photo_id: str) -> None:
return
delay = next(delays, None)
if delay is None:
logger.critical(f"[{flickr_photo_id}] Rate limit exhausted after all retries")
logger.critical(
f"[{flickr_photo_id}] Rate limit exhausted after all retries"
)
raise RateLimitExhausted
logger.warning(f"[{flickr_photo_id}] Rate limited, retrying in {delay}s")
logger.warning(
f"[{flickr_photo_id}] Rate limited, retrying in {delay}s"
)
time.sleep(delay)
except Exception as e:
logger.error(f"[{flickr_photo_id}] {e}")
Expand Down
94 changes: 94 additions & 0 deletions src/wikibots/flickrreview.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import logging
from typing import Any

from wikibots.lib.bot import BaseBot

logger = logging.getLogger(__name__)

FLICKRREVIEW_TEMPLATE = "{{Flickrreview}}\n"


class FlickreviewrBot(BaseBot):
redis_prefix = "flickrreview"
summary = (
"add [[Template:Flickrreview|Flickrreview]] template to Flickypedia uploads"
)
search_query = (
'file: incategory:"Uploads using Flickypedia" -hastemplate:Flickrreview'
)

def skip_page(self, page: dict[str, Any]) -> bool:
mid = f"M{page['pageid']}"
redis_key = f"{self.redis_prefix}:commons:{mid}"

if self.has_template(page, "FlickreviewR"):
logger.info(f"Skipping: FlickreviewR already present on {page['title']}")
self.redis.set(redis_key, 1)
return True

if self.has_user_edited(page):
logger.info(f"Skipping: bot has already edited {page['title']}")
self.redis.set(redis_key, 1)
return True

return False

def treat_page(self) -> None:
assert self.wiki_properties

self.parse_wikicode()
assert self.wiki_properties.wikicode

wikicode = self.wiki_properties.wikicode
templates = wikicode.filter_templates()

flickypedia = next(
(t for t in templates if t.name.strip() == "Uploaded with Flickypedia"),
None,
)
if flickypedia is None:
logger.warning(
f"No Flickypedia template found on {self.current_page['title']}"
)
self.redis.set(self.wiki_properties.redis_key, 1)
return

wikicode.insert_before(flickypedia, FLICKRREVIEW_TEMPLATE)

new_text = str(wikicode).replace("\n\n{{Flickrreview}}", "\n{{Flickrreview}}")

if self.dry_run:
logger.info(f"Dry run: would edit {self.current_page['title']}")
logger.info(new_text)
return

result = self._commons_api(
params={"action": "edit"},
method="POST",
data={
"pageid": self.current_page["pageid"],
"text": new_text,
"summary": self.summary,
"bot": "1",
"token": self._get_csrf_token(),
},
)
if "error" in result:
logger.critical(
f"API error editing {self.current_page['title']}: {result['error']}"
)
return

logger.info(f"Added Flickrreview to {self.current_page['title']}")
self.redis.set(self.wiki_properties.redis_key, 1)


def main() -> None:
logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
)
FlickreviewrBot().run()


if __name__ == "__main__":
main()
Loading
Loading