From 00482e5170596ab4d2c80ad4dd4893b928495597 Mon Sep 17 00:00:00 2001 From: yao Date: Wed, 13 May 2026 21:39:42 +0800 Subject: [PATCH 1/7] feat(phys_org): add Phys.org mirror site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a 16th WebHarbor mirror at https://phys.org — a science / technology / research news aggregator. Real RSS-derived catalog of 210 articles across 7 categories (Physics, Earth, Technology, Biology, Chemistry, Astronomy, Nanotechnology) with real thumbnails, plus 4 benchmark users with seeded saved articles, comments (incl. cross-user reply chains), and search history. Registered as the 16th site at port 40015. .gitignore was tightened because the previous inline-comment patterns for sites/*/scraped_data/ and sites/*/instance/ were not matching (Codex finding, fixed in this PR). Site features: - Categories with recent/popular sort - Article detail with source journal / institution / DOI - Threaded comments with reply UI (parent-article validation) - Save articles with notes (auth) - Token-overlap scored search with category filter - Trending list, user profile, account edit, login/register Determinism work for byte-identical reset: - RSS pubDate parsing strips trailing TZ token (strptime %Z rejects EDT) - Pinned bcrypt hash for benchmark users (random salt would drift md5) - Per-article RNG seeded by slug for synthesized author/journal/views - /article/ GET no longer mutates Article.views (Codex finding) Open-redirect hardening: - _safe_next() validates next= targets in /login and /save (Codex finding) Tasks: 18 WebVoyager-format tasks in sites/phys_org/tasks.jsonl, covering search, browse, detail, comment thread reading, save toggle, auth flows, and one comparison task. Assets: heavy assets (instance_seed/phys_org.db, static/images/) live in the paired HF dataset PR; phys_org.tar.gz is 460K, db md5 b4a324122c3cb0a56b8d511e73ff13a7. .assets-revision uses 'main' so the HF merge will roll in automatically. --- .gitignore | 2 +- Dockerfile | 2 +- control_server.py | 2 +- sites/phys_org/_health.py | 72 +++ sites/phys_org/app.py | 557 +++++++++++++++++++ sites/phys_org/requirements.txt | 1 + sites/phys_org/seed_data.py | 527 ++++++++++++++++++ sites/phys_org/static/css/.gitkeep | 0 sites/phys_org/static/css/main.css | 469 ++++++++++++++++ sites/phys_org/static/icons/.gitkeep | 0 sites/phys_org/static/icons/favicon.ico | Bin 0 -> 233 bytes sites/phys_org/static/icons/placeholder.svg | 9 + sites/phys_org/static/js/.gitkeep | 0 sites/phys_org/tasks.jsonl | 18 + sites/phys_org/templates/.gitkeep | 0 sites/phys_org/templates/_macros.html | 48 ++ sites/phys_org/templates/account.html | 60 ++ sites/phys_org/templates/article_detail.html | 146 +++++ sites/phys_org/templates/base.html | 66 +++ sites/phys_org/templates/category.html | 46 ++ sites/phys_org/templates/index.html | 89 +++ sites/phys_org/templates/login.html | 24 + sites/phys_org/templates/register.html | 33 ++ sites/phys_org/templates/saved.html | 37 ++ sites/phys_org/templates/search.html | 44 ++ sites/phys_org/templates/trending.html | 11 + sites/phys_org/templates/user.html | 29 + websyn_start.sh | 14 +- 28 files changed, 2297 insertions(+), 9 deletions(-) create mode 100644 sites/phys_org/_health.py create mode 100644 sites/phys_org/app.py create mode 100644 sites/phys_org/requirements.txt create mode 100644 sites/phys_org/seed_data.py create mode 100644 sites/phys_org/static/css/.gitkeep create mode 100644 sites/phys_org/static/css/main.css create mode 100644 sites/phys_org/static/icons/.gitkeep create mode 100644 sites/phys_org/static/icons/favicon.ico create mode 100644 sites/phys_org/static/icons/placeholder.svg create mode 100644 sites/phys_org/static/js/.gitkeep create mode 100644 sites/phys_org/tasks.jsonl create mode 100644 sites/phys_org/templates/.gitkeep create mode 100644 sites/phys_org/templates/_macros.html create mode 100644 sites/phys_org/templates/account.html create mode 100644 sites/phys_org/templates/article_detail.html create mode 100644 sites/phys_org/templates/base.html create mode 100644 sites/phys_org/templates/category.html create mode 100644 sites/phys_org/templates/index.html create mode 100644 sites/phys_org/templates/login.html create mode 100644 sites/phys_org/templates/register.html create mode 100644 sites/phys_org/templates/saved.html create mode 100644 sites/phys_org/templates/search.html create mode 100644 sites/phys_org/templates/trending.html create mode 100644 sites/phys_org/templates/user.html diff --git a/.gitignore b/.gitignore index 24ce1529..e7899232 100644 --- a/.gitignore +++ b/.gitignore @@ -94,4 +94,4 @@ secrets.json # ============================================================ # Agent demo results # ============================================================= -agent_demo/runs/ \ No newline at end of file +agent_demo/runs/ diff --git a/Dockerfile b/Dockerfile index 1e86b1d0..ad6d5d9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,6 @@ COPY control_server.py /opt/control_server.py COPY site_runner.py /opt/site_runner.py RUN chmod +x /opt/websyn_start.sh -EXPOSE 8101 40000-40015 +EXPOSE 8101 40000-40016 CMD ["/opt/websyn_start.sh"] diff --git a/control_server.py b/control_server.py index 4b6b995e..5c8d6cbe 100644 --- a/control_server.py +++ b/control_server.py @@ -26,7 +26,7 @@ 'allrecipes', 'amazon', 'apple', 'arxiv', 'bbc_news', 'booking', 'github', 'google_flights', 'google_map', 'google_search', 'huggingface', 'wolfram_alpha', 'cambridge_dictionary', - 'coursera', 'espn', 'merriam_webster', + 'coursera', 'espn', 'merriam_webster', 'phys_org', ] BASE_PORT = 40000 WEBSYN_DIR = '/opt/WebSyn' diff --git a/sites/phys_org/_health.py b/sites/phys_org/_health.py new file mode 100644 index 00000000..b0514f7e --- /dev/null +++ b/sites/phys_org/_health.py @@ -0,0 +1,72 @@ +"""Phys.org mirror health check.""" +from healthcheck import random_user + + +def run(p): + # 1. Home page renders + p.assert_get('home', '/', must_contain='Phys.org') + + # 2. Category pages render (DB read) + p.assert_get('category physics', '/category/physics', must_contain='Physics') + p.assert_get('category technology', '/category/technology', must_contain='Technology') + + # 3. Trending list renders + p.assert_get('trending', '/trending', must_contain='Trending') + + # 4. Search returns results (token-overlap match) + p.assert_get('search quantum', '/search?q=quantum', must_contain='quantum') + + # 5. User profile (DB read) + p.assert_get('user profile', '/user/alice_j', must_contain='alice_j') + + # 6. Article detail page (DB read; pick the first article slug from home) + home_html = p.get('/').text if hasattr(p.get('/'), 'text') else '' + # Fallback: known seed article slug pattern uses kebab; we look up by id 1. + # The home grid links to /article/; just pick a simple test that the + # detail route is wired up at all. + p.assert_get('article first', '/article/' + _first_slug(home_html, fallback='nonexistent'), + accept_status=(200, 404)) + + # 7. Register page renders (CSRF visible) + user = random_user() + html = p.assert_get('register page', '/register', must_contain='csrf_token') + token = p.csrf(html) + if not token: + p.check('register csrf token', False, 'no csrf in register form') + return + + # 8. Submit registration (DB write) + p.assert_post('register submit', '/register', { + 'csrf_token': token, + 'username': user['name'], + 'email': f"{user['name']}@test.com", + 'full_name': user['name'].title(), + 'password': user['password'], + }, accept_status=(200, 302, 303)) + + # Logout to confirm /login renders + p.get('/logout') + + # 9. Login page renders + html = p.assert_get('login page', '/login', accept_status=(200, 302, 303)) + token = p.csrf(html) if html else '' + + # 10. Submit login (DB read + session) + if token: + p.assert_post('login submit', '/login', { + 'csrf_token': token, + 'email': f"{user['name']}@test.com", + 'password': user['password'], + }, accept_status=(200, 302, 303)) + else: + p.check('login submit', True, 'already authenticated from register') + + # 11. Authenticated: account page accessible + p.assert_get('account page', '/account', accept_status=(200, 302, 303)) + + +def _first_slug(html: str, fallback: str) -> str: + """Best-effort: pull the first /article/ link from the home page.""" + import re + m = re.search(r'/article/([a-z0-9-]+)', html or '') + return m.group(1) if m else fallback diff --git a/sites/phys_org/app.py b/sites/phys_org/app.py new file mode 100644 index 00000000..1873bcaa --- /dev/null +++ b/sites/phys_org/app.py @@ -0,0 +1,557 @@ +"""Phys.org mirror — Flask application.""" +import os +import re +from datetime import datetime, timedelta +from urllib.parse import urlparse + +from flask import (Flask, render_template, request, redirect, url_for, + flash, abort, jsonify) +from flask_sqlalchemy import SQLAlchemy +from flask_login import (LoginManager, UserMixin, login_user, logout_user, + login_required, current_user) +from flask_wtf import FlaskForm +from flask_wtf.csrf import CSRFProtect +from flask_bcrypt import Bcrypt +from wtforms import StringField, PasswordField, TextAreaField, HiddenField +from wtforms.validators import DataRequired, Length, Optional, Email +from sqlalchemy import or_, desc, func +from markupsafe import Markup + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +app = Flask(__name__, instance_path=os.path.join(BASE_DIR, "instance")) +app.config['SECRET_KEY'] = 'phys-org-mirror-secret-key' +app.config['SQLALCHEMY_DATABASE_URI'] = ( + f"sqlite:///{os.path.join(BASE_DIR, 'instance', 'phys_org.db')}" +) +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False +app.config['WTF_CSRF_TIME_LIMIT'] = None + +os.makedirs(os.path.join(BASE_DIR, 'instance'), exist_ok=True) + +db = SQLAlchemy(app) +bcrypt = Bcrypt(app) +login_manager = LoginManager(app) +login_manager.login_view = 'login' +login_manager.login_message = 'Please sign in to continue.' +csrf = CSRFProtect(app) + + +# ----- Sanitize filter (for body HTML) ----- + +SAFE_TAGS = re.compile( + r'<(?!/?(?:a|p|i|b|em|strong|code|pre|br|ul|ol|li|h2|h3|blockquote)\b)[^>]+>', + re.IGNORECASE +) + + +@app.template_filter('sanitize') +def sanitize_html(text): + if not text: + return '' + cleaned = SAFE_TAGS.sub('', text) + return Markup(cleaned) + + +@app.template_filter('time_ago') +def time_ago_filter(dt): + if not dt: + return '' + return _time_ago(dt) + + +def _time_ago(dt: datetime) -> str: + now = datetime.utcnow() + diff = now - dt + seconds = int(diff.total_seconds()) + if seconds < 60: + return f"{max(seconds, 0)}s ago" + minutes = seconds // 60 + if minutes < 60: + return f"{minutes} min ago" + hours = minutes // 60 + if hours < 24: + return f"{hours} hour{'s' if hours != 1 else ''} ago" + days = hours // 24 + if days < 14: + return f"{days} day{'s' if days != 1 else ''} ago" + return dt.strftime('%b %d, %Y') + + +# ----- Models ----- + +class User(db.Model, UserMixin): + __tablename__ = 'users' + id = db.Column(db.Integer, primary_key=True) + username = db.Column(db.String(80), unique=True, nullable=False, index=True) + email = db.Column(db.String(200), unique=True, nullable=False, index=True) + password_hash = db.Column(db.String(255), nullable=False) + full_name = db.Column(db.String(200), default='') + bio = db.Column(db.Text, default='') + location = db.Column(db.String(120), default='') + interests = db.Column(db.String(255), default='') # comma-separated category slugs + created_at = db.Column(db.DateTime, default=datetime.utcnow) + + +class Category(db.Model): + __tablename__ = 'categories' + id = db.Column(db.Integer, primary_key=True) + slug = db.Column(db.String(60), unique=True, nullable=False, index=True) + name = db.Column(db.String(120), nullable=False) + description = db.Column(db.Text, default='') + sort_order = db.Column(db.Integer, default=100) + + articles = db.relationship('Article', backref='category', lazy='dynamic') + + @property + def article_count(self): + return Article.query.filter_by(category_id=self.id).count() + + +class Article(db.Model): + __tablename__ = 'articles' + id = db.Column(db.Integer, primary_key=True) + slug = db.Column(db.String(120), unique=True, nullable=False, index=True) + title = db.Column(db.String(500), nullable=False) + subtitle = db.Column(db.String(500), default='') + body = db.Column(db.Text, default='') # paragraphs separated by \n\n + author_name = db.Column(db.String(200), default='Phys.org Staff') + source_journal = db.Column(db.String(200), default='') + source_institution = db.Column(db.String(200), default='') + doi_url = db.Column(db.String(500), default='') + image_filename = db.Column(db.String(200), default='') # under static/images/ + subsection = db.Column(db.String(120), default='') # e.g., 'Optics & Photonics' + category_id = db.Column(db.Integer, db.ForeignKey('categories.id')) + published_at = db.Column(db.DateTime, default=datetime.utcnow) + views = db.Column(db.Integer, default=0) + featured = db.Column(db.Boolean, default=False) + + comments = db.relationship('Comment', backref='article', + cascade='all, delete-orphan', lazy='dynamic') + saves = db.relationship('SavedArticle', backref='article', + cascade='all, delete-orphan', lazy='dynamic') + + @property + def comment_count(self): + return self.comments.count() + + @property + def save_count(self): + return self.saves.count() + + @property + def reading_time(self): + wc = len((self.body or '').split()) + return max(1, wc // 220) + + def get_paragraphs(self): + return [p.strip() for p in re.split(r"\n\n+", self.body or '') if p.strip()] + + @property + def published_str(self): + return _time_ago(self.published_at) if self.published_at else '' + + +class Comment(db.Model): + __tablename__ = 'comments' + id = db.Column(db.Integer, primary_key=True) + text = db.Column(db.Text, nullable=False) + user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False) + article_id = db.Column(db.Integer, db.ForeignKey('articles.id'), nullable=False) + parent_id = db.Column(db.Integer, db.ForeignKey('comments.id'), nullable=True) + score = db.Column(db.Integer, default=0) + created_at = db.Column(db.DateTime, default=datetime.utcnow) + + user = db.relationship('User', backref='comments') + replies = db.relationship('Comment', backref=db.backref('parent', remote_side=[id]), + lazy='dynamic') + + @property + def time_ago(self): + return _time_ago(self.created_at) + + +class SavedArticle(db.Model): + __tablename__ = 'saved_articles' + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False, index=True) + article_id = db.Column(db.Integer, db.ForeignKey('articles.id'), nullable=False, index=True) + note = db.Column(db.String(500), default='') + created_at = db.Column(db.DateTime, default=datetime.utcnow) + __table_args__ = (db.UniqueConstraint('user_id', 'article_id'),) + + user = db.relationship('User', backref='saved') + + +class SearchHistory(db.Model): + __tablename__ = 'search_history' + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False, index=True) + query_text = db.Column('query', db.String(500), nullable=False) + created_at = db.Column(db.DateTime, default=datetime.utcnow) + + user = db.relationship('User', backref='searches') + + +# ----- Forms ----- + +class LoginForm(FlaskForm): + email = StringField('Email or username', validators=[DataRequired(), Length(3, 200)]) + password = PasswordField('Password', validators=[DataRequired()]) + + +class RegisterForm(FlaskForm): + username = StringField('Username', validators=[DataRequired(), Length(2, 80)]) + email = StringField('Email', validators=[DataRequired(), Email(), Length(3, 200)]) + full_name = StringField('Full name', validators=[Optional(), Length(0, 200)]) + password = PasswordField('Password', validators=[DataRequired(), Length(6, 128)]) + + +class ProfileForm(FlaskForm): + full_name = StringField('Full name', validators=[Optional(), Length(0, 200)]) + bio = TextAreaField('Bio', validators=[Optional(), Length(0, 2000)]) + location = StringField('Location', validators=[Optional(), Length(0, 120)]) + interests = StringField('Interests (comma separated category slugs)', + validators=[Optional(), Length(0, 255)]) + + +class CommentForm(FlaskForm): + text = TextAreaField('Comment', validators=[DataRequired(), Length(1, 2000)]) + parent_id = HiddenField() + + +class SaveForm(FlaskForm): + note = StringField('Note', validators=[Optional(), Length(0, 500)]) + + +# ----- Auth ----- + +@login_manager.user_loader +def load_user(user_id): + return db.session.get(User, int(user_id)) + + +# ----- Helpers ----- + +STOP_WORDS = {'the', 'a', 'an', 'in', 'on', 'at', 'to', 'for', 'of', 'and', + 'or', 'is', 'it', 'by', 'with', 'as', 'be', 'this', 'that', + 'are', 'was', 'were', 'from', 'how', 'what', 'why', 'we', 'i'} + + +def tokenize(query: str): + return [t.lower() for t in re.split(r'\W+', query or '') + if t.lower() not in STOP_WORDS and len(t) > 1] + + +def _safe_next(target: str | None, fallback: str) -> str: + """Return ``target`` only if it is a same-origin path on this app. + + Login and save handlers accept a `next=` parameter so the user lands + back where they came from. Without validation, an attacker could + pass `next=https://evil.example.com` and turn the site into an + open-redirect gadget. We accept only relative paths that have no + scheme/netloc, otherwise we fall back.""" + if not target: + return fallback + parsed = urlparse(target) + if parsed.scheme or parsed.netloc: + return fallback + if not target.startswith('/'): + return fallback + return target + + +def _flatten_comments(comments, depth=0): + result = [] + for c in comments: + result.append({'comment': c, 'depth': depth}) + children = c.replies.order_by(Comment.created_at).all() + result.extend(_flatten_comments(children, depth + 1)) + return result + + +@app.context_processor +def inject_globals(): + cats = Category.query.order_by(Category.sort_order, Category.name).all() + return {'all_categories': cats, 'site_name': 'Phys.org Mirror'} + + +# ----- Routes ----- + +@app.route('/') +def index(): + featured = Article.query.filter_by(featured=True) \ + .order_by(desc(Article.published_at)).limit(5).all() + latest = Article.query.order_by(desc(Article.published_at)).limit(20).all() + cats = Category.query.order_by(Category.sort_order).all() + by_cat = [] + for c in cats: + items = Article.query.filter_by(category_id=c.id) \ + .order_by(desc(Article.published_at)).limit(4).all() + if items: + by_cat.append((c, items)) + sidebar_trending = Article.query.order_by(desc(Article.views)).limit(6).all() + return render_template('index.html', featured=featured, latest=latest, + by_cat=by_cat, sidebar_trending=sidebar_trending) + + +@app.route('/category/') +def category(slug): + cat = Category.query.filter_by(slug=slug).first_or_404() + page = request.args.get('page', 1, type=int) + sort = request.args.get('sort', 'recent') + q = Article.query.filter_by(category_id=cat.id) + if sort == 'popular': + q = q.order_by(desc(Article.views), desc(Article.published_at)) + else: + q = q.order_by(desc(Article.published_at)) + pagination = q.paginate(page=page, per_page=12, error_out=False) + sidebar_trending = Article.query.order_by(desc(Article.views)).limit(6).all() + return render_template('category.html', category=cat, pagination=pagination, + sort=sort, sidebar_trending=sidebar_trending) + + +@app.route('/article/') +def article_detail(slug): + art = Article.query.filter_by(slug=slug).first_or_404() + # Note: we deliberately do NOT increment views on GET. `views` is the + # seeded popularity signal used by trending/popular sort and by + # benchmark tasks (Phys.org--3, --10, --15). Mutating it on every page + # view would let an agent's browsing order shift task answers and + # would break /reset/ byte-identity. If a future task needs a + # runtime visit counter, add a separate column for that. + top_comments = Comment.query.filter_by(article_id=art.id, parent_id=None) \ + .order_by(Comment.created_at).all() + comment_tree = _flatten_comments(top_comments) + related = Article.query.filter(Article.category_id == art.category_id, + Article.id != art.id) \ + .order_by(desc(Article.published_at)).limit(4).all() + is_saved = False + if current_user.is_authenticated: + is_saved = SavedArticle.query.filter_by( + user_id=current_user.id, article_id=art.id).first() is not None + form = CommentForm() + save_form = SaveForm() + return render_template('article_detail.html', article=art, comment_tree=comment_tree, + related=related, form=form, save_form=save_form, + is_saved=is_saved) + + +@app.route('/article//comment', methods=['POST']) +@login_required +def post_comment(slug): + art = Article.query.filter_by(slug=slug).first_or_404() + form = CommentForm() + if not form.validate_on_submit(): + flash('Comment could not be posted.', 'error') + return redirect(url_for('article_detail', slug=slug)) + + parent_id = None + raw_parent = (form.parent_id.data or '').strip() + if raw_parent: + try: + candidate = int(raw_parent) + except ValueError: + flash('Invalid reply target.', 'error') + return redirect(url_for('article_detail', slug=slug)) + parent = db.session.get(Comment, candidate) + # Reject replies whose parent doesn't exist or belongs to a different + # article — prevents cross-article reply injection via crafted forms. + if parent is None or parent.article_id != art.id: + flash('Invalid reply target.', 'error') + return redirect(url_for('article_detail', slug=slug)) + parent_id = candidate + + c = Comment(text=form.text.data.strip(), user_id=current_user.id, + article_id=art.id, parent_id=parent_id) + db.session.add(c) + db.session.commit() + flash('Comment posted.', 'success') + return redirect(url_for('article_detail', slug=slug) + f'#comment-{c.id}') + + +@app.route('/save/', methods=['POST']) +@login_required +def save_article(article_id): + art = Article.query.get_or_404(article_id) + existing = SavedArticle.query.filter_by( + user_id=current_user.id, article_id=art.id).first() + form = SaveForm() + if existing: + db.session.delete(existing) + db.session.commit() + flash('Removed from your saved list.', 'info') + else: + note = form.note.data.strip() if form.note.data else '' + s = SavedArticle(user_id=current_user.id, article_id=art.id, note=note) + db.session.add(s) + db.session.commit() + flash('Article saved.', 'success') + next_url = _safe_next(request.form.get('next'), + url_for('article_detail', slug=art.slug)) + return redirect(next_url) + + +@app.route('/saved') +@login_required +def saved(): + items = SavedArticle.query.filter_by(user_id=current_user.id) \ + .order_by(desc(SavedArticle.created_at)).all() + return render_template('saved.html', items=items) + + +@app.route('/trending') +def trending(): + page = request.args.get('page', 1, type=int) + pagination = Article.query.order_by(desc(Article.views), desc(Article.published_at)) \ + .paginate(page=page, per_page=15, error_out=False) + return render_template('trending.html', pagination=pagination) + + +@app.route('/search') +def search(): + q = (request.args.get('q') or '').strip() + page = request.args.get('page', 1, type=int) + cat_filter = (request.args.get('category') or '').strip() + + if not q: + return render_template('search.html', query='', results=[], page=1, + total=0, has_next=False, has_prev=False, + selected_category=cat_filter) + + if current_user.is_authenticated: + sh = SearchHistory(user_id=current_user.id, query_text=q) + db.session.add(sh) + db.session.commit() + + tokens = tokenize(q) + if not tokens: + return render_template('search.html', query=q, results=[], page=1, + total=0, has_next=False, has_prev=False, + selected_category=cat_filter) + + base = Article.query + if cat_filter: + cat = Category.query.filter_by(slug=cat_filter).first() + if cat: + base = base.filter(Article.category_id == cat.id) + + filters = [] + for token in tokens: + like = f'%{token}%' + filters.append(or_(Article.title.ilike(like), + Article.subtitle.ilike(like), + Article.body.ilike(like))) + candidates = base.filter(or_(*filters)).limit(800).all() + + scored = [] + for art in candidates: + blob = f"{art.title}\n{art.subtitle}\n{art.body}".lower() + score = sum(1 for t in tokens if t in blob) + if score > 0: + scored.append((art, score)) + scored.sort(key=lambda x: (-x[1], + -(x[0].published_at.timestamp() if x[0].published_at else 0))) + + per_page = 12 + total = len(scored) + start = (page - 1) * per_page + end = start + per_page + page_items = [a for a, _ in scored[start:end]] + return render_template('search.html', query=q, results=page_items, page=page, + total=total, has_next=end < total, has_prev=page > 1, + selected_category=cat_filter) + + +@app.route('/user/') +def user_profile(username): + u = User.query.filter_by(username=username).first_or_404() + saved_count = SavedArticle.query.filter_by(user_id=u.id).count() + comment_count = Comment.query.filter_by(user_id=u.id).count() + recent_comments = Comment.query.filter_by(user_id=u.id) \ + .order_by(desc(Comment.created_at)).limit(10).all() + return render_template('user.html', user=u, saved_count=saved_count, + comment_count=comment_count, recent_comments=recent_comments) + + +@app.route('/account', methods=['GET', 'POST']) +@login_required +def account(): + form = ProfileForm(obj=current_user) + if form.validate_on_submit(): + current_user.full_name = form.full_name.data or '' + current_user.bio = form.bio.data or '' + current_user.location = form.location.data or '' + current_user.interests = form.interests.data or '' + db.session.commit() + flash('Profile updated.', 'success') + return redirect(url_for('account')) + history = SearchHistory.query.filter_by(user_id=current_user.id) \ + .order_by(desc(SearchHistory.created_at)).limit(20).all() + return render_template('account.html', form=form, search_history=history) + + +@app.route('/login', methods=['GET', 'POST']) +def login(): + if current_user.is_authenticated: + return redirect(url_for('index')) + form = LoginForm() + if form.validate_on_submit(): + user = User.query.filter( + (User.email == form.email.data) | (User.username == form.email.data) + ).first() + if user and bcrypt.check_password_hash(user.password_hash, form.password.data): + login_user(user) + next_page = _safe_next(request.args.get('next'), + url_for('index')) + return redirect(next_page) + flash('Invalid email or password.', 'error') + return render_template('login.html', form=form) + + +@app.route('/register', methods=['GET', 'POST']) +def register(): + if current_user.is_authenticated: + return redirect(url_for('index')) + form = RegisterForm() + if form.validate_on_submit(): + if User.query.filter_by(email=form.email.data).first(): + flash('Email already registered.', 'error') + elif User.query.filter_by(username=form.username.data).first(): + flash('Username already taken.', 'error') + else: + pw = bcrypt.generate_password_hash(form.password.data).decode('utf-8') + u = User(username=form.username.data, email=form.email.data, + full_name=form.full_name.data or '', password_hash=pw) + db.session.add(u) + db.session.commit() + login_user(u) + return redirect(url_for('index')) + return render_template('register.html', form=form) + + +@app.route('/logout') +@login_required +def logout(): + logout_user() + return redirect(url_for('index')) + + +@app.route('/_health') +def _health(): + return {'ok': True, 'site': 'phys_org'} + + +# ----- Seed bootstrap ----- + +from seed_data import seed_database, seed_benchmark_users # noqa: E402 + +with app.app_context(): + db.create_all() + seed_database(db, User, Category, Article, Comment, bcrypt) + seed_benchmark_users(db, User, Category, Article, Comment, SavedArticle, SearchHistory, bcrypt) + + +if __name__ == '__main__': + port = int(os.environ.get('PORT', 5000)) + app.run(host='0.0.0.0', port=port, debug=False) diff --git a/sites/phys_org/requirements.txt b/sites/phys_org/requirements.txt new file mode 100644 index 00000000..e3e9a71d --- /dev/null +++ b/sites/phys_org/requirements.txt @@ -0,0 +1 @@ +Flask diff --git a/sites/phys_org/seed_data.py b/sites/phys_org/seed_data.py new file mode 100644 index 00000000..4b97e6a7 --- /dev/null +++ b/sites/phys_org/seed_data.py @@ -0,0 +1,527 @@ +"""Phys.org mirror — idempotent seed data. + +Loads ``scraped_data/phys_data.json`` (real RSS-derived articles) and synthesizes +the side data agents need: source journals/institutions, additional body text, +benchmark users with saved articles + comments + search history. + +The byte-identical reset invariant requires that each ``seed_*`` function is a +no-op when the DB is already populated. Per-row gates aren't enough — even an +empty ``commit()`` bumps SQLite metadata. +""" +import json +import os +import random +import re +from datetime import datetime, timedelta + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DATA_FILE = os.path.join(BASE_DIR, 'scraped_data', 'phys_data.json') + +# Pinned reference date so "published_at" values are stable across rebuilds and +# the byte-identical reset invariant holds. +MIRROR_REFERENCE_DATE = datetime(2026, 5, 12, 12, 0, 0) + + +CATEGORIES = [ + ('physics', 'Physics', + 'Latest news in physics, materials science, optics, quantum and superconductivity.', 10), + ('earth', 'Earth Sciences', + 'Climate, geology, oceanography and the planet that supports us.', 20), + ('technology', 'Technology', + 'AI, robotics, computing, energy, and engineering breakthroughs.', 30), + ('biology', 'Biology', + 'Cell biology, ecology, evolution, plants and animals.', 40), + ('chemistry', 'Chemistry', + 'Molecules, reactions, materials and analytical chemistry.', 50), + ('astronomy', 'Astronomy & Space', + 'Cosmology, planetary science, missions and space exploration.', 60), + ('nanotechnology', 'Nanotechnology', + 'Nanomaterials, nanoelectronics, bio- and nano-technology.', 70), + ('other', 'Other Sciences', + 'Mathematics, social sciences, archaeology and education.', 80), +] + + +# Pools used to synthesize plausible journal / institution data per category. +# Real phys.org articles cite these journals heavily; using them keeps the +# detail page realistic. Each tuple is (journal, parent publisher). +JOURNALS_BY_CATEGORY = { + 'physics': [ + 'Physical Review Letters', 'Nature Physics', 'Physical Review B', + 'Reviews of Modern Physics', 'New Journal of Physics', + 'Physical Review Applied', 'Optics Express', 'Nature Photonics', + ], + 'earth': [ + 'Nature Geoscience', 'Geophysical Research Letters', + 'Journal of Climate', 'Earth and Planetary Science Letters', + 'Nature Climate Change', 'Geology', 'Journal of Geophysical Research: Atmospheres', + ], + 'technology': [ + 'Nature Electronics', 'IEEE Transactions on Robotics', + 'ACM Computing Surveys', 'Joule', 'Energy & Environmental Science', + 'Nature Machine Intelligence', 'Science Robotics', + ], + 'biology': [ + 'Cell', 'Nature', 'Current Biology', 'Proceedings of the National Academy of Sciences', + 'eLife', 'Nature Ecology & Evolution', 'PLOS Biology', 'Molecular Ecology', + ], + 'chemistry': [ + 'Journal of the American Chemical Society', 'Nature Chemistry', + 'Angewandte Chemie International Edition', 'ACS Central Science', + 'Chemical Science', 'Inorganic Chemistry', + ], + 'astronomy': [ + 'The Astrophysical Journal', 'Monthly Notices of the Royal Astronomical Society', + 'Astronomy & Astrophysics', 'Nature Astronomy', 'Icarus', + 'Astrophysical Journal Letters', + ], + 'nanotechnology': [ + 'Nature Nanotechnology', 'ACS Nano', 'Nano Letters', + 'Advanced Materials', 'Small', 'npj 2D Materials and Applications', + ], + 'other': [ + 'Journal of Archaeological Science', 'Nature Human Behaviour', + 'PNAS', 'Science Advances', 'PLOS ONE', 'Proceedings of the Royal Society B', + ], +} + + +INSTITUTIONS_BY_CATEGORY = { + 'physics': [ + 'Massachusetts Institute of Technology', 'Stanford University', + 'CERN', 'University of Cambridge', 'ETH Zurich', 'Caltech', + 'Max Planck Institute for Quantum Optics', 'Technion', + 'Princeton University', 'Argonne National Laboratory', + ], + 'earth': [ + 'NOAA', 'University of Washington', 'Scripps Institution of Oceanography', + 'University of Oxford', 'Potsdam Institute for Climate Impact Research', + 'Woods Hole Oceanographic Institution', 'NASA Goddard Space Flight Center', + 'Columbia University', + ], + 'technology': [ + 'Carnegie Mellon University', 'Google DeepMind', 'IBM Research', + 'University of California, Berkeley', 'University of Toronto', + 'EPFL', 'Microsoft Research', 'KAIST', 'Tsinghua University', + ], + 'biology': [ + 'Harvard Medical School', 'University of Oxford', + 'Howard Hughes Medical Institute', 'EMBL-EBI', + 'Salk Institute', 'University of Tokyo', 'Wellcome Sanger Institute', + 'University of Pennsylvania', + ], + 'chemistry': [ + 'Northwestern University', 'University of Chicago', + 'University of California, Los Angeles', 'Scripps Research', + 'University of Bristol', 'Tokyo Institute of Technology', + 'Imperial College London', + ], + 'astronomy': [ + 'NASA Jet Propulsion Laboratory', 'European Southern Observatory', + 'Space Telescope Science Institute', 'Harvard-Smithsonian Center for Astrophysics', + 'Max Planck Institute for Astronomy', 'Caltech', 'University of Arizona', + ], + 'nanotechnology': [ + 'KAIST', 'Rice University', 'IBM Research – Zurich', + 'National University of Singapore', 'University of Manchester', + 'Tsinghua University', 'Lawrence Berkeley National Laboratory', + ], + 'other': [ + 'University of Oxford', 'Max Planck Institute for the Science of Human History', + 'University of Chicago', 'London School of Economics', + 'University of Cape Town', 'Hebrew University of Jerusalem', + ], +} + + +# Synthetic body filler. Only used when the RSS description is too short. +GENERIC_PARAGRAPHS = [ + "The findings, the team writes, open new questions about how robust the underlying assumptions of the field really are, and suggest that further independent replications will be needed before the wider community converges on a single explanation.", + "Beyond the immediate result, the work hints at practical applications. The authors caution, however, that translating these laboratory observations into deployable systems is likely to take several more years of engineering effort and additional safety review.", + "Independent researchers not involved in the study described the data as 'compelling' and 'a useful starting point,' while noting that some of the boldest claims will need to be tested in larger and more diverse samples before being accepted as established science.", +] + + +def _slugify(text: str, maxlen: int = 70) -> str: + s = re.sub(r"[^a-zA-Z0-9]+", "-", text or "").strip("-").lower() + return s[:maxlen] or "article" + + +def _parse_pub(s: str) -> datetime: + """Parse RSS pubDate. Falls back to MIRROR_REFERENCE_DATE. + + strptime's %Z only accepts UTC/GMT and the local TZ on most platforms, so + real RSS dates like 'EDT' / 'PDT' don't parse. Strip the trailing zone + word (or +0000-style offset) and parse the remainder.""" + if not s: + return MIRROR_REFERENCE_DATE + s = s.strip() + m = re.match(r'(.+?\d{2}:\d{2}:\d{2})\s*\S+', s) + base = m.group(1) if m else s + for fmt in ("%a, %d %b %Y %H:%M:%S", + "%a, %d %b %Y %H:%M", + "%a, %d %b %Y"): + try: + return datetime.strptime(base.strip(), fmt) + except Exception: + continue + return MIRROR_REFERENCE_DATE + + +def _strip_html(text: str) -> str: + text = re.sub(r"<[^>]+>", "", text or "") + text = re.sub(r"\s+", " ", text).strip() + return text + + +def _build_body(rss_desc: str, title: str, *, rng: random.Random) -> str: + """Return paragraph-separated body text. Use the RSS description as the + lede and append synthetic-but-plausible follow-on paragraphs so each + article has at least 3 paragraphs.""" + lede = _strip_html(rss_desc) or title + paragraphs = [lede] + pool = GENERIC_PARAGRAPHS[:] + rng.shuffle(pool) + paragraphs.append(pool[0]) + paragraphs.append(pool[1]) + return "\n\n".join(paragraphs) + + +def seed_database(db, User, Category, Article, Comment, bcrypt): + if Article.query.count() > 0: + return + + # Seed categories first (only if empty — gated by the outer check on + # Article, but we double-check here to keep the function self-contained). + cat_id_map = {} + for slug, name, desc, order in CATEGORIES: + c = Category.query.filter_by(slug=slug).first() + if c is None: + c = Category(slug=slug, name=name, description=desc, sort_order=order) + db.session.add(c) + db.session.flush() + cat_id_map[slug] = c.id + + if not os.path.exists(DATA_FILE): + # No scraped data — bail without committing anything else, leaving + # only categories. (The reset invariant still holds because we did + # commit categories on the first call; subsequent calls are gated.) + db.session.commit() + return + + with open(DATA_FILE) as f: + items = json.load(f) + + rng = random.Random(20260513) + + # Determine featured article ids ahead of time so the same items are + # picked across rebuilds. + item_keys = [it.get('link') or it.get('title') for it in items] + featured_count = min(8, len(items)) + featured_keys = set(rng.sample(item_keys, featured_count)) if item_keys else set() + + next_id = 1 + seen_slugs = set() + for it in items: + title = (it.get('title') or '').strip() + if not title: + continue + slug = it.get('slug') or _slugify(title) + original = slug + n = 2 + while slug in seen_slugs: + slug = f"{original}-{n}" + n += 1 + seen_slugs.add(slug) + + cat_slug = it.get('category_slug') or 'other' + if cat_slug not in cat_id_map: + cat_slug = 'other' + cat_id = cat_id_map[cat_slug] + + published = _parse_pub(it.get('pub_date') or '') + # Subsection from RSS categories (e.g. "Optics & Photonics") + rss_cats = it.get('rss_categories') or [] + subsection = (rss_cats[0] if rss_cats else '').strip() + + # Author: real RSS dc:creator if present, else synthesized. + author_real = (it.get('author') or '').strip() + if author_real: + author_name = author_real + else: + # Reproducible synthesized author per article slug. + r2 = random.Random(slug + ':author') + firsts = ['Sarah', 'Michael', 'Ananya', 'Jorge', 'Mei', 'David', + 'Priya', 'Liam', 'Fatima', 'Hiroshi', 'Olivia', 'Karim', + 'Nina', 'Oluwa', 'Bjorn', 'Elena'] + lasts = ['Patel', 'Garcia', 'Nguyen', 'Kowalski', 'Rossi', 'Tanaka', + 'Andersen', 'Okafor', 'Singh', 'Yamamoto', 'Hernandez', + 'Mueller', 'Ahmed', 'Park'] + author_name = f"{r2.choice(firsts)} {r2.choice(lasts)}" + + # Journal / institution synthesized per article (deterministic by slug) + r3 = random.Random(slug + ':source') + journal = r3.choice(JOURNALS_BY_CATEGORY.get(cat_slug, JOURNALS_BY_CATEGORY['other'])) + institution = r3.choice(INSTITUTIONS_BY_CATEGORY.get(cat_slug, INSTITUTIONS_BY_CATEGORY['other'])) + # DOI: synthesize a stable but fake-looking DOI per article id. + doi = f"https://doi.org/10.{1000 + next_id}/phys.{published.year}.{next_id:05d}" + + body = _build_body(it.get('description') or '', title, rng=rng) + subtitle = _strip_html(it.get('description') or '')[:240] + + image_filename = it.get('local_image') or '' + + # Deterministic view counts so trending lists are stable across + # rebuilds (only changes when new articles are added). Range chosen + # to give a clear winner: ~1500-9000 with one popular article in + # each category capped near the top. + rv = random.Random(slug + ':views') + views = rv.randint(150, 9000) + + is_featured = (it.get('link') or it.get('title')) in featured_keys + + art = Article( + id=next_id, + slug=slug, + title=title, + subtitle=subtitle, + body=body, + author_name=author_name, + source_journal=journal, + source_institution=institution, + doi_url=doi, + image_filename=image_filename, + subsection=subsection, + category_id=cat_id, + published_at=published, + views=views, + featured=is_featured, + ) + db.session.add(art) + next_id += 1 + + db.session.commit() + + +# --------------------------------------------------------------------------- +# Benchmark users +# --------------------------------------------------------------------------- + +BENCH_USERS = [ + dict(username='alice_j', email='alice.j@test.com', full_name='Alice Johnson', + bio='PhD student in astrophysics. Saving everything about exoplanets and dark matter.', + location='Boston, MA', interests='astronomy,physics'), + dict(username='bob_c', email='bob.c@test.com', full_name='Bob Chen', + bio='Climate-tech reporter. Following ocean carbon, methane and renewables stories.', + location='Seattle, WA', interests='earth,technology'), + dict(username='carol_d', email='carol.d@test.com', full_name='Carol Davis', + bio='Computational biologist. Long-time fan of CRISPR, protein design and ecology.', + location='Cambridge, UK', interests='biology,chemistry'), + dict(username='david_k', email='david.k@test.com', full_name='David Kim', + bio='Materials engineer. Reads everything tagged Nanotechnology, Optics & Photonics.', + location='Seoul, South Korea', interests='nanotechnology,physics'), +] +PASSWORD = 'TestPass123!' + +# Pre-generated bcrypt hash for PASSWORD. bcrypt.generate_password_hash uses a +# random salt on every call, which would break the byte-identical reset +# invariant — so we pin one valid hash here. Verified at boot time by +# bcrypt.check_password_hash; rotate by running: +# from flask_bcrypt import Bcrypt; from flask import Flask +# print(Bcrypt(Flask(__name__)).generate_password_hash('TestPass123!').decode()) +PINNED_PASSWORD_HASH = ( + '$2b$12$zV7HfiJmZTqLsgP30kyvJemamXfJyBv66FPuQOrwYXXsyQvrafvie' +) + + +# Stable user-id mapping: 1001..1004 (well above article-derived ids so we +# don't collide with any future re-numbering). +USER_ID_BASE = 1001 + + +def _pick_articles(Article, *, where: dict, n: int, seed: str) -> list: + """Return up to n articles matching ``where`` filters, deterministically + ordered by id so the result is identical across rebuilds.""" + q = Article.query + for k, v in where.items(): + q = q.filter(getattr(Article, k) == v) + items = q.order_by(Article.id).all() + rng = random.Random(seed) + rng.shuffle(items) + return items[:n] + + +def seed_benchmark_users(db, User, Category, Article, Comment, SavedArticle, SearchHistory, bcrypt): + if User.query.filter_by(email='alice.j@test.com').first(): + return + + # Categories must exist (created by seed_database). Look up ids. + pw_hash = PINNED_PASSWORD_HASH + + user_objs = {} + for i, u in enumerate(BENCH_USERS): + obj = User( + id=USER_ID_BASE + i, + username=u['username'], + email=u['email'], + full_name=u['full_name'], + bio=u['bio'], + location=u['location'], + interests=u['interests'], + password_hash=pw_hash, + created_at=MIRROR_REFERENCE_DATE - timedelta(days=180 + i * 30), + ) + db.session.add(obj) + user_objs[u['username']] = obj + db.session.flush() + + # Save articles aligned to each user's interests so saved-list tasks have + # depth and disambiguation candidates. + save_targets = { + 'alice_j': [ + ('astronomy', 4), + ('physics', 2), + ], + 'bob_c': [ + ('earth', 4), + ('technology', 2), + ], + 'carol_d': [ + ('biology', 4), + ('chemistry', 2), + ], + 'david_k': [ + ('nanotechnology', 3), + ('physics', 2), + ], + } + next_save_id = 1 + save_notes_by_user = { + 'alice_j': ['Read for thesis chapter 3', 'Cite in proposal', 'Follow-up reading', + 'Discuss with advisor', 'Seminar candidate', 'Review for journal club'], + 'bob_c': ['Story idea — angle 2', 'Lead source candidate', 'Background reading', + 'Quote for upcoming feature', 'Verify with NOAA contact', 'Pitch to editor'], + 'carol_d': ['Methods section', 'Lab meeting share', 'Forward to postdocs', + 'Compare with our pipeline', 'Re-read after deadline', 'Class material'], + 'david_k': ['Material spec lookup', 'Patent landscape', 'Contact authors', + 'Internal report cite', 'Compare with our process', 'Lab notebook ref'], + } + for username, plan in save_targets.items(): + u = user_objs[username] + notes = save_notes_by_user[username] + used = 0 + for cat_slug, n in plan: + cat = Category.query.filter_by(slug=cat_slug).first() + if cat is None: + continue + articles = _pick_articles(Article, where={'category_id': cat.id}, n=n, + seed=f"{username}:save:{cat_slug}") + for art in articles: + sa = SavedArticle( + id=next_save_id, + user_id=u.id, + article_id=art.id, + note=notes[used % len(notes)], + created_at=MIRROR_REFERENCE_DATE - timedelta(days=2 + used * 3), + ) + db.session.add(sa) + next_save_id += 1 + used += 1 + + # Comments per user (2-4 each) on a deterministic spread of articles. + comments_plan = { + 'alice_j': [ + 'Beautiful explanation of the dark-matter constraints — the figure 3 plot is doing a lot of work here.', + 'Worth comparing with the 2024 Planck re-analysis — different priors but converging conclusions.', + 'Saving this for the journal club tomorrow; the methodology section is a great teaching example.', + ], + 'bob_c': [ + 'This contradicts the line a senator pushed last week. Sourcing this for my Wednesday column.', + 'The institution statement and the paper itself disagree on the 2030 timeline. Anyone seen the PRR?', + 'Modeling assumptions feel optimistic, but the data underlying them is solid. Cautious thumbs up.', + ], + 'carol_d': [ + 'The CRISPR off-target rates here are an order of magnitude lower than what we see in our pipeline.', + 'I love that they released the raw sequencing data. Re-running their analysis tonight.', + 'Nice work, but I expected more discussion of polyploid edge cases.', + ], + 'david_k': [ + 'The fabrication tolerance is the real story here, not the zero-resistance claim.', + 'Anyone have access to the SI? The thickness vs. mobility curve is the only thing that matters.', + 'Calling it now: this technique will be in commercial sensors by 2028.', + ], + } + next_comment_id = 1 + for username, comment_texts in comments_plan.items(): + u = user_objs[username] + # Pick articles whose category matches the user's first interest tag, + # so a "comments by alice on physics articles" task is well-defined. + first_interest = u.interests.split(',')[0] + cat = Category.query.filter_by(slug=first_interest).first() + if cat is None: + target_articles = Article.query.order_by(Article.id).limit(len(comment_texts)).all() + else: + target_articles = _pick_articles(Article, where={'category_id': cat.id}, + n=len(comment_texts), + seed=f"{username}:comment") + for i, art in enumerate(target_articles): + c = Comment( + id=next_comment_id, + text=comment_texts[i], + user_id=u.id, + article_id=art.id, + parent_id=None, + score=0, + created_at=MIRROR_REFERENCE_DATE - timedelta(days=1 + i * 4), + ) + db.session.add(c) + next_comment_id += 1 + + # Seed a few cross-user reply chains so commenter-thread tasks work. + reply_seeds = [ + ('bob_c', 'alice_j', 0, 'Totally agree on the priors point — the new constraint is much tighter though.'), + ('alice_j', 'carol_d', 0, 'The polyploid section was a missed opportunity, you are right.'), + ('david_k', 'bob_c', 1, 'I think the institution is hedging because of an unannounced pilot — keep watching.'), + ] + for replier_username, target_username, target_idx, text in reply_seeds: + replier = user_objs[replier_username] + target_user = user_objs[target_username] + target_comments = Comment.query.filter_by(user_id=target_user.id) \ + .order_by(Comment.id).all() + if target_idx >= len(target_comments): + continue + parent = target_comments[target_idx] + c = Comment( + id=next_comment_id, + text=text, + user_id=replier.id, + article_id=parent.article_id, + parent_id=parent.id, + score=0, + created_at=parent.created_at + timedelta(hours=6), + ) + db.session.add(c) + next_comment_id += 1 + + # Search history per user (2-3 each) + search_plan = { + 'alice_j': ['exoplanet atmosphere', 'dark matter halo', 'james webb'], + 'bob_c': ['ocean carbon capture', 'methane emissions arctic'], + 'carol_d': ['CRISPR off-target', 'protein structure prediction', 'mitochondria'], + 'david_k': ['2D material superconductor', 'graphene transistor'], + } + next_sh_id = 1 + for username, queries in search_plan.items(): + u = user_objs[username] + for j, q in enumerate(queries): + sh = SearchHistory( + id=next_sh_id, + user_id=u.id, + query_text=q, + created_at=MIRROR_REFERENCE_DATE - timedelta(days=1 + j * 2, + hours=j * 5), + ) + db.session.add(sh) + next_sh_id += 1 + + db.session.commit() diff --git a/sites/phys_org/static/css/.gitkeep b/sites/phys_org/static/css/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/sites/phys_org/static/css/main.css b/sites/phys_org/static/css/main.css new file mode 100644 index 00000000..3aaf34c7 --- /dev/null +++ b/sites/phys_org/static/css/main.css @@ -0,0 +1,469 @@ +/* Phys.org mirror styles — clean white bg, deep navy header, blue accents. */ + +:root { + --c-text: #1a1a1a; + --c-muted: #6b6b6b; + --c-link: #0a4ea2; + --c-link-hover: #062f63; + --c-navy: #16285b; + --c-navy-dark: #0c1a3e; + --c-accent: #0e6cc1; + --c-bg: #ffffff; + --c-card: #ffffff; + --c-border: #e3e6ea; + --c-soft: #f5f7fa; + --c-warn: #c0392b; + --c-success: #2c7a3a; +} + +* { box-sizing: border-box; } + +html, body { + margin: 0; + padding: 0; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", + Arial, "Noto Sans", sans-serif; + font-size: 15px; + line-height: 1.5; + color: var(--c-text); + background: var(--c-bg); +} + +a { color: var(--c-link); text-decoration: none; } +a:hover { color: var(--c-link-hover); text-decoration: underline; } + +img { max-width: 100%; height: auto; display: block; } + +/* ---- Header ---- */ + +.site-header { + background: var(--c-navy); + color: #fff; + border-bottom: 3px solid var(--c-accent); +} +.site-header a { color: #fff; } +.site-header a:hover { color: #cfe1ff; text-decoration: none; } + +.header-top { + display: flex; + align-items: center; + padding: 12px 20px; + max-width: 1200px; + margin: 0 auto; + gap: 18px; +} +.brand { + font-size: 26px; + font-weight: 800; + letter-spacing: -0.5px; +} +.brand .dot { color: var(--c-accent); } +.tagline { + color: #cdd6e6; + font-size: 13px; + margin-left: 4px; +} +.header-search { + flex: 1; + max-width: 500px; + margin-left: auto; +} +.header-search form { display: flex; gap: 0; } +.header-search input[type=text], +.header-search input[type=search] { + flex: 1; + padding: 8px 12px; + border: 1px solid var(--c-navy-dark); + border-radius: 4px 0 0 4px; + font-size: 14px; + outline: none; +} +.header-search button { + padding: 8px 14px; + background: var(--c-accent); + color: #fff; + border: none; + border-radius: 0 4px 4px 0; + cursor: pointer; + font-weight: 600; +} +.header-account { + display: flex; + gap: 12px; + font-size: 13px; + white-space: nowrap; +} + +.nav-bar { + background: var(--c-navy-dark); + font-size: 13px; +} +.nav-bar ul { + list-style: none; + display: flex; + flex-wrap: wrap; + margin: 0 auto; + padding: 0 20px; + max-width: 1200px; +} +.nav-bar li a { + display: block; + padding: 10px 14px; + color: #e6ecf7; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; +} +.nav-bar li a:hover { background: var(--c-accent); color: #fff; } +.nav-bar li a.active { background: var(--c-accent); color: #fff; } + +/* ---- Layout ---- */ + +.container { + max-width: 1200px; + margin: 0 auto; + padding: 20px; +} +.layout { + display: grid; + grid-template-columns: minmax(0, 1fr) 320px; + gap: 28px; +} +@media (max-width: 900px) { + .layout { grid-template-columns: 1fr; } +} + +/* ---- Cards & lists ---- */ + +.section-heading { + display: flex; + align-items: baseline; + gap: 12px; + margin: 28px 0 14px; + padding-bottom: 6px; + border-bottom: 2px solid var(--c-navy); +} +.section-heading h2 { + margin: 0; + font-size: 18px; + color: var(--c-navy); + text-transform: uppercase; + letter-spacing: 0.5px; +} +.section-heading a.see-all { font-size: 13px; } + +.article-card { + display: grid; + grid-template-columns: 160px minmax(0, 1fr); + gap: 16px; + padding: 14px 0; + border-bottom: 1px solid var(--c-border); +} +.article-card .thumb { + width: 160px; + height: 110px; + overflow: hidden; + border-radius: 4px; + background: var(--c-soft); +} +.article-card .thumb img { width: 100%; height: 100%; object-fit: cover; } +.article-card .body { min-width: 0; } +.article-card h3 { + margin: 0 0 6px; + font-size: 17px; + line-height: 1.3; +} +.article-card h3 a { color: var(--c-text); } +.article-card h3 a:hover { color: var(--c-link); } +.article-card .meta { + font-size: 12px; + color: var(--c-muted); + margin-bottom: 6px; +} +.article-card .meta .tag { + display: inline-block; + background: var(--c-soft); + color: var(--c-navy); + padding: 2px 8px; + border-radius: 3px; + font-weight: 600; + text-transform: uppercase; + font-size: 11px; + margin-right: 6px; +} +.article-card .summary { color: #444; font-size: 14px; } + +.featured-grid { + display: grid; + grid-template-columns: 2fr 1fr 1fr; + gap: 16px; + margin: 12px 0 24px; +} +@media (max-width: 800px) { .featured-grid { grid-template-columns: 1fr; } } +.feat-main, .feat-side { + background: #fff; + border: 1px solid var(--c-border); + border-radius: 4px; + overflow: hidden; +} +.feat-main .thumb { height: 280px; background: var(--c-soft); } +.feat-side .thumb { height: 130px; background: var(--c-soft); } +.feat-main .thumb img, +.feat-side .thumb img { width: 100%; height: 100%; object-fit: cover; } +.feat-main .pad { padding: 14px 16px 18px; } +.feat-side .pad { padding: 10px 12px 14px; } +.feat-main h2, .feat-side h3 { margin: 4px 0 6px; line-height: 1.25; } +.feat-main h2 { font-size: 22px; } +.feat-main h2 a, .feat-side h3 a { color: var(--c-text); } +.feat-main h2 a:hover, .feat-side h3 a:hover { color: var(--c-link); } + +/* ---- Sidebar ---- */ + +.sidebar { font-size: 14px; } +.sidebar .widget { + background: var(--c-soft); + border: 1px solid var(--c-border); + border-radius: 4px; + padding: 14px 16px; + margin-bottom: 18px; +} +.sidebar .widget h3 { + margin: 0 0 10px; + font-size: 14px; + color: var(--c-navy); + text-transform: uppercase; + letter-spacing: 0.5px; + border-bottom: 1px solid var(--c-border); + padding-bottom: 6px; +} +.sidebar ol, .sidebar ul { + margin: 0; + padding-left: 18px; +} +.sidebar li { margin-bottom: 8px; line-height: 1.35; } + +/* ---- Article detail ---- */ + +.article-detail { + background: #fff; +} +.article-detail .crumbs { + font-size: 13px; + color: var(--c-muted); + margin-bottom: 8px; +} +.article-detail h1 { + margin: 6px 0 8px; + font-size: 30px; + line-height: 1.2; + color: var(--c-text); +} +.article-detail .subtitle { + font-size: 17px; + color: #333; + margin: 0 0 14px; + line-height: 1.4; +} +.article-detail .byline { + font-size: 13px; + color: var(--c-muted); + margin-bottom: 14px; + border-bottom: 1px solid var(--c-border); + padding-bottom: 12px; +} +.article-detail .byline strong { color: #333; } +.article-detail .hero-image { + margin: 0 0 16px; + border-radius: 4px; + overflow: hidden; + background: var(--c-soft); +} +.article-detail .hero-image img { width: 100%; height: auto; } +.article-detail .body p { + margin: 0 0 14px; + font-size: 16px; + line-height: 1.65; +} +.source-block { + margin: 22px 0; + padding: 14px 16px; + background: var(--c-soft); + border-left: 4px solid var(--c-accent); + border-radius: 3px; + font-size: 14px; +} +.source-block dt { + display: inline-block; + font-weight: 700; + width: 130px; + color: var(--c-navy); +} +.source-block dd { display: inline; margin: 0; } +.source-block dl > div { margin-bottom: 6px; } + +.action-bar { + display: flex; + gap: 10px; + margin: 16px 0; + padding: 10px 0; + border-top: 1px solid var(--c-border); + border-bottom: 1px solid var(--c-border); +} +.btn { + display: inline-block; + padding: 7px 14px; + background: var(--c-accent); + color: #fff; + border: 1px solid transparent; + border-radius: 3px; + cursor: pointer; + font-size: 14px; + font-weight: 600; +} +.btn:hover { background: var(--c-navy); color: #fff; text-decoration: none; } +.btn.secondary { background: #fff; color: var(--c-navy); border-color: var(--c-navy); } +.btn.secondary:hover { background: var(--c-navy); color: #fff; } +.btn.danger { background: var(--c-warn); } + +/* ---- Comments ---- */ + +.comments-section { margin-top: 32px; } +.comments-section h2 { + font-size: 18px; + color: var(--c-navy); + border-bottom: 2px solid var(--c-navy); + padding-bottom: 6px; + text-transform: uppercase; + letter-spacing: 0.5px; +} +.comment { + border-left: 3px solid var(--c-border); + padding: 8px 0 8px 12px; + margin: 8px 0; +} +.comment .head { + font-size: 13px; + color: var(--c-muted); + margin-bottom: 4px; +} +.comment .head a.author { font-weight: 700; color: var(--c-navy); } +.comment .body { font-size: 15px; line-height: 1.45; } +.comment-form textarea { + width: 100%; + min-height: 100px; + padding: 10px; + border: 1px solid var(--c-border); + border-radius: 4px; + font: inherit; +} + +/* ---- Forms ---- */ + +.form-card { + max-width: 480px; + margin: 30px auto; + padding: 26px 28px; + background: #fff; + border: 1px solid var(--c-border); + border-radius: 4px; + box-shadow: 0 2px 6px rgba(15, 30, 75, 0.04); +} +.form-card h1 { + margin: 0 0 16px; + font-size: 22px; + color: var(--c-navy); +} +.form-card .field { margin-bottom: 14px; } +.form-card label { + display: block; + font-size: 13px; + font-weight: 600; + margin-bottom: 4px; + color: #333; +} +.form-card input[type=text], .form-card input[type=email], +.form-card input[type=password], .form-card textarea { + width: 100%; + padding: 8px 10px; + border: 1px solid var(--c-border); + border-radius: 3px; + font: inherit; +} +.form-card .errors { color: var(--c-warn); font-size: 13px; } +.form-card .actions { margin-top: 18px; } +.form-card .alt { font-size: 13px; margin-top: 14px; color: var(--c-muted); } + +.flash { + padding: 10px 14px; + margin: 0 0 14px; + border-radius: 3px; + font-size: 14px; +} +.flash-success { background: #e2f6e8; color: var(--c-success); border: 1px solid #b9e2c4; } +.flash-error { background: #fdecea; color: var(--c-warn); border: 1px solid #f5c2bb; } +.flash-info { background: #e6f1fb; color: var(--c-link); border: 1px solid #c2dbf2; } + +/* ---- Pagination ---- */ + +.pagination { + margin: 22px 0; + display: flex; + gap: 6px; + align-items: center; +} +.pagination .page, +.pagination .arrow { + display: inline-block; + padding: 5px 11px; + border: 1px solid var(--c-border); + border-radius: 3px; + font-size: 13px; + color: var(--c-link); + background: #fff; +} +.pagination .page.active { + background: var(--c-navy); + color: #fff; + border-color: var(--c-navy); +} +.pagination .arrow.disabled { + color: #aaa; + background: var(--c-soft); + pointer-events: none; +} + +/* ---- Footer ---- */ + +.site-footer { + background: var(--c-navy-dark); + color: #cfd6e6; + font-size: 13px; + padding: 20px; + margin-top: 36px; +} +.site-footer .container { display: flex; justify-content: space-between; flex-wrap: wrap; gap: 12px; } +.site-footer a { color: #cfd6e6; } +.site-footer a:hover { color: #fff; } + +/* ---- Misc ---- */ + +.text-muted { color: var(--c-muted); font-size: 13px; } +.tag-pill { + display: inline-block; + font-size: 11px; + padding: 2px 8px; + background: var(--c-accent); + color: #fff; + border-radius: 3px; + text-transform: uppercase; + font-weight: 700; + letter-spacing: 0.4px; +} +.profile-head { + background: var(--c-soft); + padding: 18px 20px; + border-radius: 4px; + margin-bottom: 20px; +} +.profile-head h1 { margin: 0 0 4px; color: var(--c-navy); } +.profile-stats { display: flex; gap: 18px; font-size: 14px; color: var(--c-muted); } diff --git a/sites/phys_org/static/icons/.gitkeep b/sites/phys_org/static/icons/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/sites/phys_org/static/icons/favicon.ico b/sites/phys_org/static/icons/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..2beaea967cb03a435e868239de35a4690fd96d23 GIT binary patch literal 233 zcmZQzU<5)11qL8cV7Lq<#eldoz|WnRONtA~u;CS$`znE(TJwJY5_^ zD&{2rnSab7_<&3FV@WsN1c{f03M-EW)X!`bT(;~LPxBPMP~D2U-iI$m*(MtZL|>o% z@M~;adiaVgMrP*2*VNe7XH8JI(U#n1GAYYSP|Rlkxerf-oX_~QmG?WyDamdS2-~!@ zbKyrR=e=TQgD#$)&0}xZ>=6~|Ah}>CYtQ1NUqAGxEDPl3XuPPP#NpQ5p&-Gl#OB1n Zuy&WyF6k1>Pe4~Pc)I$ztaD0e0sxJ)PNM(- literal 0 HcmV?d00001 diff --git a/sites/phys_org/static/icons/placeholder.svg b/sites/phys_org/static/icons/placeholder.svg new file mode 100644 index 00000000..b8bf5b88 --- /dev/null +++ b/sites/phys_org/static/icons/placeholder.svg @@ -0,0 +1,9 @@ + + + + + + + phys.org + diff --git a/sites/phys_org/static/js/.gitkeep b/sites/phys_org/static/js/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/sites/phys_org/tasks.jsonl b/sites/phys_org/tasks.jsonl new file mode 100644 index 00000000..2b5f8287 --- /dev/null +++ b/sites/phys_org/tasks.jsonl @@ -0,0 +1,18 @@ +{"web_name": "Phys.org", "id": "Phys.org--0", "ques": "Find the article 'Magnetic checkerboard separates microparticles by size and sends them along different paths' in the Physics category and report which journal it cites as its source.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--1", "ques": "Open the article on phys.org titled 'Quantum circuit test finally exposes what has been warping performance' and report the institution that is listed as 'Provided by'.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--2", "ques": "Search for 'quantum' on phys.org. Among the matching results, find the article whose title mentions 'a trillionth of a billionth of a joule' and report its source journal.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--3", "ques": "Browse the Trending page on phys.org and report the author name shown on the article currently ranked #1 by views.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--4", "ques": "Sign in as alice.j@test.com (password TestPass123!), open the user's saved article list, and report how many of those saved articles belong to the 'Astronomy & Space' category.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--5", "ques": "Sign in as bob.c@test.com (password TestPass123!), open the saved article whose user note is 'Pitch to editor', and report the article's source journal.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--6", "ques": "Sign in as carol.d@test.com (password TestPass123!) and post a new top-level comment with the text 'Reviewed for our weekly journal club' on any article in the Biology category, then report the article's title.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--7", "ques": "Sign in as david.k@test.com (password TestPass123!), find a Nanotechnology article you have not yet saved, save it with the note 'Compare with our process', then verify it appears in your saved list and report the article title.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--8", "ques": "Open the public profile page for user 'carol_d' on phys.org and report the count of their public comments shown in the profile stats.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--9", "ques": "Search the phys.org mirror for 'graphene' and report how many results are returned in total (the count shown next to the search term).", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--10", "ques": "On the phys.org Astronomy & Space category page, sort by 'Popular' and report the title of the most-viewed article in that category.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--11", "ques": "Compare the publication dates of the two phys.org articles 'Magnetic checkerboard separates microparticles by size and sends them along different paths' and 'Quantum geometry applied to light-based systems expands toolkit for topological photonics'. Report which one was published earlier.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--12", "ques": "Find the comment thread on the phys.org article 'JWST spots two early black holes growing far faster than their galaxies' where user bob_c replied to a top-level comment by alice_j. Report the full text of bob_c's reply.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--13", "ques": "Register a new account on phys.org with username 'qa_explorer', email 'qa_explorer@example.com', full name 'QA Explorer', password 'BenchmarkPass2026'. Then open Account Settings, set the Location field to 'Berlin, Germany', and save. Report the username that appears in the page header after saving.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--14", "ques": "Sign in as alice.j@test.com (password TestPass123!), open the article 'How a single star can reshape an entire galaxy', and remove it from her saved articles. Then visit the saved-articles page and report (a) how many items remain in the saved list and (b) the title of the most-recently-saved article shown at the top of the list.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--15", "ques": "On the phys.org homepage there is a 'Trending now' sidebar widget. Report the title and view count of the third entry in that sidebar list.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--16", "ques": "On phys.org, search for 'CO2' and use the category filter to restrict to Chemistry. Among the filtered results, identify the article that mentions 'polyionic liquids' and report its source journal.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name": "Phys.org", "id": "Phys.org--17", "ques": "Sign in as alice.j@test.com (password TestPass123!), open Account Settings, and report the most recent search query shown in the user's recent search history widget.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} diff --git a/sites/phys_org/templates/.gitkeep b/sites/phys_org/templates/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/sites/phys_org/templates/_macros.html b/sites/phys_org/templates/_macros.html new file mode 100644 index 00000000..e80de24c --- /dev/null +++ b/sites/phys_org/templates/_macros.html @@ -0,0 +1,48 @@ +{% macro article_card(a) -%} + +{%- endmacro %} + +{% macro pager(pagination, endpoint, kw={}) -%} +{% if pagination.pages > 1 %} + +{% endif %} +{%- endmacro %} diff --git a/sites/phys_org/templates/account.html b/sites/phys_org/templates/account.html new file mode 100644 index 00000000..8053aa9d --- /dev/null +++ b/sites/phys_org/templates/account.html @@ -0,0 +1,60 @@ +{% extends 'base.html' %} +{% block title %}Account settings — Phys.org Mirror{% endblock %} +{% block content %} +
+
+

Account settings

+
+ {{ form.csrf_token }} +
+ + +
+
+ + +
+
+ + {{ form.full_name(size=40) }} +
+
+ + {{ form.location(size=40) }} +
+
+ + {{ form.bio(rows=4, cols=50) }} +
+
+ + {{ form.interests(size=50) }} +
+
+
+
+ +
+{% endblock %} diff --git a/sites/phys_org/templates/article_detail.html b/sites/phys_org/templates/article_detail.html new file mode 100644 index 00000000..62ef1bdc --- /dev/null +++ b/sites/phys_org/templates/article_detail.html @@ -0,0 +1,146 @@ +{% extends 'base.html' %} +{% from '_macros.html' import article_card %} +{% block title %}{{ article.title }} — Phys.org Mirror{% endblock %} +{% block content %} +
+
+
+ Home + {% if article.category %} + / {{ article.category.name }} + {% endif %} + {% if article.subsection %} / {{ article.subsection }}{% endif %} +
+ +

{{ article.title }}

+ {% if article.subtitle %}

{{ article.subtitle }}

{% endif %} + +
+ {{ article.author_name }} + · Published {{ article.published_at.strftime('%b %d, %Y') if article.published_at else '' }} + · {{ article.reading_time }} min read + · {{ article.views }} view{{ '' if article.views == 1 else 's' }} +
+ + {% if article.image_filename %} +
+ +
+ {% endif %} + +
+ {% for p in article.get_paragraphs() %} +

{{ p|sanitize }}

+ {% endfor %} +
+ +
+
+ {% if article.source_journal %} +
Journal
{{ article.source_journal }}
+ {% endif %} + {% if article.source_institution %} +
Provided by
{{ article.source_institution }}
+ {% endif %} + {% if article.doi_url %} + + {% endif %} +
+
+ +
+ {% if current_user.is_authenticated %} +
+ {{ save_form.csrf_token }} + + {% if is_saved %} + + {% else %} + + + {% endif %} +
+ {% else %} + Sign in to save + {% endif %} + {{ article.comment_count }} comment{{ '' if article.comment_count == 1 else 's' }} +
+ +
+

Comments

+ {% if comment_tree %} + {% for entry in comment_tree %} + {% set c = entry.comment %} +
+
+ {{ c.user.username }} + · {{ c.time_ago }} + {% if current_user.is_authenticated %} + · Reply + {% endif %} +
+
{{ c.text }}
+
+ {% endfor %} + {% else %} +

No comments yet.

+ {% endif %} + + {% if current_user.is_authenticated %} +
+ {{ form.csrf_token }} + + + +
+
+ + {% else %} +

Sign in to comment.

+ {% endif %} +
+
+ + +
+{% endblock %} diff --git a/sites/phys_org/templates/base.html b/sites/phys_org/templates/base.html new file mode 100644 index 00000000..17434846 --- /dev/null +++ b/sites/phys_org/templates/base.html @@ -0,0 +1,66 @@ + + + + + +{% block title %}{{ site_name }}{% endblock %} + + + + +
+
+ phys.orgmirror + Science • Technology • Research news +
+
+ + +
+
+ +
+ +
+ +
+ {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for cat, msg in messages %} +
{{ msg }}
+ {% endfor %} + {% endif %} + {% endwith %} + {% block content %}{% endblock %} +
+ +
+
+
© Phys.org Mirror — for benchmark research, not affiliated with phys.org
+
+ Home · + Trending · + Search +
+
+
+ + diff --git a/sites/phys_org/templates/category.html b/sites/phys_org/templates/category.html new file mode 100644 index 00000000..e95463f5 --- /dev/null +++ b/sites/phys_org/templates/category.html @@ -0,0 +1,46 @@ +{% extends 'base.html' %} +{% from '_macros.html' import article_card, pager %} +{% block title %}{{ category.name }} — Phys.org Mirror{% endblock %} +{% block content %} +
+
+
+

{{ category.name }}

+ {{ pagination.total }} article{{ '' if pagination.total == 1 else 's' }} + + Recent · + Popular + +
+ {% if category.description %}

{{ category.description }}

{% endif %} + {% for a in pagination.items %}{{ article_card(a) }}{% endfor %} + {% if not pagination.items %}

No articles in this category yet.

{% endif %} + {{ pager(pagination, 'category', {'slug': category.slug, 'sort': sort}) }} +
+ +
+{% endblock %} diff --git a/sites/phys_org/templates/index.html b/sites/phys_org/templates/index.html new file mode 100644 index 00000000..fe7bb989 --- /dev/null +++ b/sites/phys_org/templates/index.html @@ -0,0 +1,89 @@ +{% extends 'base.html' %} +{% from '_macros.html' import article_card %} +{% block title %}Phys.org Mirror — Science, Technology, Research news{% endblock %} + +{% block content %} + +{% if featured %} +
+ {% set main = featured[0] %} + +
+ {% if main.image_filename %} + + {% else %} + + {% endif %} +
+
+ {{ main.category.name if main.category else 'Featured' }} +

{{ main.title }}

+
{{ main.published_str }}{% if main.subsection %} · {{ main.subsection }}{% endif %}
+
+ + {% for a in featured[1:5] %} + +
+ {% if a.image_filename %} + + {% else %} + + {% endif %} +
+
+ {{ a.category.name if a.category else '' }} +

{{ a.title }}

+
{{ a.published_str }}
+
+ + {% endfor %} +
+{% endif %} + +
+
+
+

Latest News

+ All recent → +
+ {% for a in latest %}{{ article_card(a) }}{% endfor %} + + {% for cat, items in by_cat %} +
+

{{ cat.name }}

+ More in {{ cat.name }} → +
+ {% for a in items %}{{ article_card(a) }}{% endfor %} + {% endfor %} +
+ +
+{% endblock %} diff --git a/sites/phys_org/templates/login.html b/sites/phys_org/templates/login.html new file mode 100644 index 00000000..1f533898 --- /dev/null +++ b/sites/phys_org/templates/login.html @@ -0,0 +1,24 @@ +{% extends 'base.html' %} +{% block title %}Sign in — Phys.org Mirror{% endblock %} +{% block content %} +
+

Sign in

+
+ {{ form.csrf_token }} +
+ + {{ form.email(size=40) }} + {% if form.email.errors %}
{{ form.email.errors[0] }}
{% endif %} +
+
+ + {{ form.password(size=40) }} + {% if form.password.errors %}
{{ form.password.errors[0] }}
{% endif %} +
+
+ +
+
No account? Create one.
+
+
+{% endblock %} diff --git a/sites/phys_org/templates/register.html b/sites/phys_org/templates/register.html new file mode 100644 index 00000000..858e6820 --- /dev/null +++ b/sites/phys_org/templates/register.html @@ -0,0 +1,33 @@ +{% extends 'base.html' %} +{% block title %}Create account — Phys.org Mirror{% endblock %} +{% block content %} +
+

Create account

+
+ {{ form.csrf_token }} +
+ + {{ form.username(size=40) }} + {% if form.username.errors %}
{{ form.username.errors[0] }}
{% endif %} +
+
+ + {{ form.email(size=40) }} + {% if form.email.errors %}
{{ form.email.errors[0] }}
{% endif %} +
+
+ + {{ form.full_name(size=40) }} +
+
+ + {{ form.password(size=40) }} + {% if form.password.errors %}
{{ form.password.errors[0] }}
{% endif %} +
+
+ +
+
Already have an account? Sign in.
+
+
+{% endblock %} diff --git a/sites/phys_org/templates/saved.html b/sites/phys_org/templates/saved.html new file mode 100644 index 00000000..d61146b2 --- /dev/null +++ b/sites/phys_org/templates/saved.html @@ -0,0 +1,37 @@ +{% extends 'base.html' %} +{% block title %}Saved articles — Phys.org Mirror{% endblock %} +{% block content %} +
+

Your saved articles

+ {{ items|length }} item{{ '' if items|length == 1 else 's' }} +
+{% if items %} + {% for it in items %} + {% set a = it.article %} + + {% endfor %} +{% else %} +

You haven't saved any articles yet. Browse the homepage and click "Save article" on any story.

+{% endif %} +{% endblock %} diff --git a/sites/phys_org/templates/search.html b/sites/phys_org/templates/search.html new file mode 100644 index 00000000..16cba6ef --- /dev/null +++ b/sites/phys_org/templates/search.html @@ -0,0 +1,44 @@ +{% extends 'base.html' %} +{% from '_macros.html' import article_card %} +{% block title %}Search — Phys.org Mirror{% endblock %} +{% block content %} +
+

Search

+ {% if query %}{{ total }} result{{ '' if total == 1 else 's' }} for "{{ query }}"{% endif %} +
+ +
+ + + +
+ +{% if query %} + {% if results %} + {% for a in results %}{{ article_card(a) }}{% endfor %} + + {% else %} +

No results matched your search. Try fewer keywords or a different category.

+ {% endif %} +{% else %} +

Type a query above to search across {{ all_categories|length }} categories.

+{% endif %} +{% endblock %} diff --git a/sites/phys_org/templates/trending.html b/sites/phys_org/templates/trending.html new file mode 100644 index 00000000..c12ca8e1 --- /dev/null +++ b/sites/phys_org/templates/trending.html @@ -0,0 +1,11 @@ +{% extends 'base.html' %} +{% from '_macros.html' import article_card, pager %} +{% block title %}Trending — Phys.org Mirror{% endblock %} +{% block content %} +
+

Trending articles

+ Sorted by total views. +
+{% for a in pagination.items %}{{ article_card(a) }}{% endfor %} +{{ pager(pagination, 'trending') }} +{% endblock %} diff --git a/sites/phys_org/templates/user.html b/sites/phys_org/templates/user.html new file mode 100644 index 00000000..b9cfd9d1 --- /dev/null +++ b/sites/phys_org/templates/user.html @@ -0,0 +1,29 @@ +{% extends 'base.html' %} +{% block title %}{{ user.username }} — Phys.org Mirror{% endblock %} +{% block content %} +
+

{{ user.full_name or user.username }}

+
@{{ user.username }}{% if user.location %} · {{ user.location }}{% endif %} · joined {{ user.created_at.strftime('%b %Y') if user.created_at else '' }}
+ {% if user.bio %}

{{ user.bio }}

{% endif %} +
+ {{ saved_count }} saved + {{ comment_count }} comments + {% if user.interests %}Interests: {{ user.interests }}{% endif %} +
+
+ +

Recent comments

+{% if recent_comments %} + {% for c in recent_comments %} +
+
+ on {{ c.article.title }} + · {{ c.time_ago }} +
+
{{ c.text }}
+
+ {% endfor %} +{% else %} +

No comments yet.

+{% endif %} +{% endblock %} diff --git a/websyn_start.sh b/websyn_start.sh index 4d690d78..596bb675 100644 --- a/websyn_start.sh +++ b/websyn_start.sh @@ -1,11 +1,11 @@ #!/bin/bash -# WebSyn startup: launch all 16 mirror sites, then exec the original CMD. +# WebSyn startup: launch all mirror sites, then exec the original CMD. # This preserves the base image's browser env server (port 8100) as PID 1. set -e SITES=(allrecipes amazon apple arxiv bbc_news booking github google_flights google_map google_search huggingface wolfram_alpha - cambridge_dictionary coursera espn merriam_webster) + cambridge_dictionary coursera espn merriam_webster phys_org) BASE_PORT=40000 PID_DIR=/tmp/websyn_pids mkdir -p "$PID_DIR" @@ -17,7 +17,9 @@ for d in "${SITES[@]}"; do cp -a "/opt/WebSyn/$d/instance_seed" "/opt/WebSyn/$d/instance" done -echo "[WebSyn] Starting 16 sites on ports ${BASE_PORT}-$((BASE_PORT + 15))..." +SITE_COUNT=${#SITES[@]} +END_PORT=$((BASE_PORT + SITE_COUNT - 1)) +echo "[WebSyn] Starting ${SITE_COUNT} sites on ports ${BASE_PORT}-${END_PORT}..." for i in "${!SITES[@]}"; do site="${SITES[$i]}" port=$((BASE_PORT + i)) @@ -51,8 +53,8 @@ except Exception: exit(1) ready=$((ready + 1)) fi done - echo " [${elapsed}/${max_wait}s] ${ready}/16 sites ready" - if [ $ready -eq 16 ]; then + echo " [${elapsed}/${max_wait}s] ${ready}/${SITE_COUNT} sites ready" + if [ $ready -eq $SITE_COUNT ]; then break fi done @@ -78,6 +80,6 @@ done echo "[WebSyn] Starting control server on :8101 (PID 1)..." # Control server becomes PID 1 — receives SIGTERM on `docker stop`, -# keeps the container alive as long as it's running. The 16 site +# keeps the container alive as long as it's running. The site # subprocesses are managed via /tmp/websyn_pids/.pid. exec python3 /opt/control_server.py --port 8101 From b350eeb1245a777f49c3ae4fb155530c224a4881 Mon Sep 17 00:00:00 2001 From: Zexu Jin Date: Sat, 15 Aug 2026 15:27:08 +0800 Subject: [PATCH 2/7] feat(phys_org): add deterministic task verifiers Add verifier and rubric coverage for all 18 tasks, harden the easy comparison tasks, and cover positive, no-op, shortcut, wrong-answer, and state-mismatch cases. Fix reviewed environment nits in asset packing, seed taxonomy and journal realism, dependencies, and site-count documentation. --- .claude/skills/review-env/SKILL.md | 10 +- AGENTS.md | 10 +- CONTRIBUTING.md | 4 +- Dockerfile | 2 +- README.md | 8 +- scripts/extract_assets.sh | 4 +- sites/phys_org/app.py | 7 +- sites/phys_org/requirements.txt | 8 + sites/phys_org/seed_data.py | 63 ++++- sites/phys_org/tasks.jsonl | 36 +-- .../verify/test_environment_quality.py | 75 +++++ sites/phys_org/verify/test_verifiers.py | 258 ++++++++++++++++++ sites/phys_org/verify/verify_0.py | 13 + sites/phys_org/verify/verify_1.py | 11 + sites/phys_org/verify/verify_10.py | 14 + sites/phys_org/verify/verify_11.py | 15 + sites/phys_org/verify/verify_12.py | 12 + sites/phys_org/verify/verify_13.py | 27 ++ sites/phys_org/verify/verify_14.py | 42 +++ sites/phys_org/verify/verify_15.py | 18 ++ sites/phys_org/verify/verify_16.py | 15 + sites/phys_org/verify/verify_17.py | 28 ++ sites/phys_org/verify/verify_2.py | 13 + sites/phys_org/verify/verify_3.py | 13 + sites/phys_org/verify/verify_4.py | 11 + sites/phys_org/verify/verify_5.py | 14 + sites/phys_org/verify/verify_6.py | 36 +++ sites/phys_org/verify/verify_7.py | 38 +++ sites/phys_org/verify/verify_8.py | 9 + sites/phys_org/verify/verify_9.py | 20 ++ sites/phys_org/verify/verify_lib.py | 182 ++++++++++++ 31 files changed, 972 insertions(+), 44 deletions(-) create mode 100644 sites/phys_org/verify/test_environment_quality.py create mode 100644 sites/phys_org/verify/test_verifiers.py create mode 100644 sites/phys_org/verify/verify_0.py create mode 100644 sites/phys_org/verify/verify_1.py create mode 100644 sites/phys_org/verify/verify_10.py create mode 100644 sites/phys_org/verify/verify_11.py create mode 100644 sites/phys_org/verify/verify_12.py create mode 100644 sites/phys_org/verify/verify_13.py create mode 100644 sites/phys_org/verify/verify_14.py create mode 100644 sites/phys_org/verify/verify_15.py create mode 100644 sites/phys_org/verify/verify_16.py create mode 100644 sites/phys_org/verify/verify_17.py create mode 100644 sites/phys_org/verify/verify_2.py create mode 100644 sites/phys_org/verify/verify_3.py create mode 100644 sites/phys_org/verify/verify_4.py create mode 100644 sites/phys_org/verify/verify_5.py create mode 100644 sites/phys_org/verify/verify_6.py create mode 100644 sites/phys_org/verify/verify_7.py create mode 100644 sites/phys_org/verify/verify_8.py create mode 100644 sites/phys_org/verify/verify_9.py create mode 100644 sites/phys_org/verify/verify_lib.py diff --git a/.claude/skills/review-env/SKILL.md b/.claude/skills/review-env/SKILL.md index 52dc9640..136ba007 100644 --- a/.claude/skills/review-env/SKILL.md +++ b/.claude/skills/review-env/SKILL.md @@ -33,18 +33,18 @@ gh pr checkout ./scripts/fetch_assets.sh # pull the pinned HF revision ./scripts/build.sh webharbor:dev docker run -d --rm --name wh-review \ - -p 8201:8101 -p 41000-41015:40000-40015 webharbor:dev + -p 8201:8101 -p 41000-41016:40000-40016 webharbor:dev ``` -Confirm the new/changed site is on the expected port (40000 + index). Note: the image now runs 16 sites (40000-40015). +Confirm the new/changed site is on the expected port (40000 + index). Note: the image now runs 17 sites (40000-40016). ### Step 2: The mechanical checks (5 minutes) Run the same Pre-PR checks the contributor was supposed to run. ```bash -# 1. all 16 sites return 200 -for p in $(seq 41000 41015); do +# 1. all 17 sites return 200 +for p in $(seq 41000 41016); do curl -so /dev/null -w "$p:%{http_code}\n" http://localhost:$p/ done @@ -231,7 +231,7 @@ Leave a structured comment on the PR: ## Review: ### Mechanical checks: PASS / FAIL -- [x] All 15 sites return 200 +- [x] All 17 sites return 200 - [x] Control plane healthy - [x] Byte-identical reset (md5 match) - [x] Parallel reset <10s diff --git a/AGENTS.md b/AGENTS.md index afa6f8b4..ad618907 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ A coding agent (Claude Code, Cursor, Aider, Codex, ...) is reading this. Read on ## What it is -15 Flask mirror websites (Amazon, GitHub, BBC News, ...) packaged into one Docker image, plus a control plane on `:8101` for resetting per-site state. Used as a deterministic offline environment for web-agent benchmarks. ~3 GB image. +17 Flask mirror websites (Amazon, GitHub, BBC News, ...) packaged into one Docker image, plus a control plane on `:8101` for resetting per-site state. Used as a deterministic offline environment for web-agent benchmarks. ~3 GB image. Two repos: - **code** (this one) — Flask apps, control plane, scripts. @@ -48,17 +48,17 @@ Inside the image, sites live at `/opt/WebSyn//`. The path predates the ren # fresh clone ./scripts/fetch_assets.sh # pulls assets from HF ./scripts/build.sh # docker build -t webharbor:dev . -docker run -d -p 8101:8101 -p 40000-40014:40000-40014 webharbor:dev +docker run -d -p 8101:8101 -p 40000-40016:40000-40016 webharbor:dev ``` Or use the published image directly: ```bash -docker run -d -p 8101:8101 -p 40000-40014:40000-40014 \ +docker run -d -p 8101:8101 -p 40000-40016:40000-40016 \ battalion7244/webharbor:latest ``` -Sites are on `40000`-`40014` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane: +Sites are on `40000`-`40016` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane: | Method | Path | Purpose | |--------|---------------------|-------------------------------------------| @@ -136,7 +136,7 @@ python3 -m py_compile sites//app.py # 3. run on alt ports (don't collide with anything you already have running) docker run -d --rm --name wh-test \ - -p 8201:8101 -p 41000-41014:40000-40014 webharbor:dev + -p 8201:8101 -p 41000-41016:40000-40016 webharbor:dev # 4. control plane healthy, all sites alive curl -s http://localhost:8201/health | python3 -m json.tool | head diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 927ed53f..7a251c56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ git clone https://github.com//webharbor && cd webharbor ./scripts/fetch_assets.sh # pull current assets ./scripts/new_site.py mywebsite # OR edit an existing site ./scripts/build.sh && docker run -d --rm \ - -p 8101:8101 -p 40000-40014:40000-40014 webharbor:dev + -p 8101:8101 -p 40000-40016:40000-40016 webharbor:dev # iterate locally... ./scripts/extract_assets.sh ../webharbor-static-pr/ # split assets out @@ -277,4 +277,4 @@ Sites must not import from one another. The image launches each as an independen ### Don't hard-code secrets -Each site sets `SECRET_KEY` to a deterministic dev value. Acceptable for a benchmark image (resets blow away sessions anyway). If a contrib ever needs real secrets, raise it in an issue first. \ No newline at end of file +Each site sets `SECRET_KEY` to a deterministic dev value. Acceptable for a benchmark image (resets blow away sessions anyway). If a contrib ever needs real secrets, raise it in an issue first. diff --git a/Dockerfile b/Dockerfile index ad6d5d9f..43088ffb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # WebHarbor — slim, self-contained image. -# 16 Flask mirror sites + control plane on :8101. +# 17 Flask mirror sites + control plane on :8101. FROM python:3.12-slim-bookworm diff --git a/README.md b/README.md index dce3f934..22675195 100644 --- a/README.md +++ b/README.md @@ -36,17 +36,17 @@ WebHarbor takes a different approach. We leverage coding agent (e.g., Claude Cod - **Deep features unlocked** — carts, checkouts, accounts, all fully testable - **Evolving** — harder tasks drive richer mirrors; the environment grows with agents - **RL-ready** — sub-second database resets between rollouts -- **Community-driven** — 15 sites today, scaling to 100+ together +- **Community-driven** — 17 sites today, scaling to 100+ together ## 🚀 Quickstart One command to run all web environments: ```bash -docker run -p 8101:8101 -p 40000-40014:40000-40014 battalion7244/webharbor:latest +docker run -p 8101:8101 -p 40000-40016:40000-40016 battalion7244/webharbor:latest ``` -Then point your agent at `http://localhost:40000` through `http://localhost:40014` to explore 15 local mirrors of webvoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, and ESPN`. +Then point your agent at `http://localhost:40000` through `http://localhost:40016` to explore 17 local mirrors of webvoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, ESPN, Merriam-Webster, and Phys.org`. For sub-second reset between rollouts, expose the control plane and call `/reset/`: @@ -111,4 +111,4 @@ WebHarbor is initiated by UNC-Chapel Hill and Microsoft, with contributions from url = {https://aiming-lab.github.io/webharbor.github.io}, note = {Project website.} } -``` \ No newline at end of file +``` diff --git a/scripts/extract_assets.sh b/scripts/extract_assets.sh index b9e58724..1dd5e8b0 100755 --- a/scripts/extract_assets.sh +++ b/scripts/extract_assets.sh @@ -42,7 +42,9 @@ for site_dir in sites/*/; do fi out="$TARGET/$site.tar.gz" - tar -czf "$out" -C sites "${members[@]}" + # macOS may synthesize AppleDouble ``._*`` metadata while archiving files. + # Exclude it explicitly so uploaded assets are portable and reproducible. + COPYFILE_DISABLE=1 tar --exclude='._*' -czf "$out" -C sites "${members[@]}" sz=$(du -sh "$out" 2>/dev/null | cut -f1) printf " %-22s -> %-30s %s\n" "$site" "$site.tar.gz" "$sz" count=$((count + 1)) diff --git a/sites/phys_org/app.py b/sites/phys_org/app.py index 1873bcaa..aba18440 100644 --- a/sites/phys_org/app.py +++ b/sites/phys_org/app.py @@ -1,11 +1,10 @@ """Phys.org mirror — Flask application.""" import os import re -from datetime import datetime, timedelta +from datetime import datetime from urllib.parse import urlparse -from flask import (Flask, render_template, request, redirect, url_for, - flash, abort, jsonify) +from flask import Flask, render_template, request, redirect, url_for, flash from flask_sqlalchemy import SQLAlchemy from flask_login import (LoginManager, UserMixin, login_user, logout_user, login_required, current_user) @@ -14,7 +13,7 @@ from flask_bcrypt import Bcrypt from wtforms import StringField, PasswordField, TextAreaField, HiddenField from wtforms.validators import DataRequired, Length, Optional, Email -from sqlalchemy import or_, desc, func +from sqlalchemy import or_, desc from markupsafe import Markup BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/sites/phys_org/requirements.txt b/sites/phys_org/requirements.txt index e3e9a71d..a28b4a40 100644 --- a/sites/phys_org/requirements.txt +++ b/sites/phys_org/requirements.txt @@ -1 +1,9 @@ Flask +Flask-SQLAlchemy +Flask-Login +Flask-WTF +Flask-Bcrypt +Werkzeug +SQLAlchemy +WTForms +email-validator diff --git a/sites/phys_org/seed_data.py b/sites/phys_org/seed_data.py index 4b97e6a7..9cd19263 100644 --- a/sites/phys_org/seed_data.py +++ b/sites/phys_org/seed_data.py @@ -37,8 +37,6 @@ 'Cosmology, planetary science, missions and space exploration.', 60), ('nanotechnology', 'Nanotechnology', 'Nanomaterials, nanoelectronics, bio- and nano-technology.', 70), - ('other', 'Other Sciences', - 'Mathematics, social sciences, archaeology and education.', 80), ] @@ -86,6 +84,61 @@ } +# Technology covers unrelated disciplines, so its RSS subsection is a better +# signal than the broad category when synthesizing a source journal. +JOURNALS_BY_SUBSECTION = { + ('technology', 'Automotive'): [ + 'IEEE Transactions on Intelligent Transportation Systems', + 'Transportation Research Part C: Emerging Technologies', + 'International Journal of Automotive Technology', + ], + ('technology', 'Consumer & Gadgets'): [ + 'IEEE Consumer Electronics Magazine', + 'IEEE Transactions on Consumer Electronics', + 'Personal and Ubiquitous Computing', + ], + ('technology', 'Electronics & Semiconductors'): [ + 'Nature Electronics', 'IEEE Electron Device Letters', + 'Advanced Electronic Materials', + ], + ('technology', 'Energy & Green Tech'): [ + 'Joule', 'Energy & Environmental Science', 'Nature Energy', + ], + ('technology', 'Engineering'): [ + 'AIAA Journal', 'Aerospace Science and Technology', + 'Advanced Engineering Materials', 'Nature Communications', + ], + ('technology', 'Internet'): [ + 'IEEE Internet Computing', 'Computer Networks', + 'ACM Transactions on Internet Technology', + ], + ('technology', 'Machine learning & AI'): [ + 'Nature Machine Intelligence', 'Journal of Machine Learning Research', + 'IEEE Transactions on Pattern Analysis and Machine Intelligence', + ], + ('technology', 'Robotics'): [ + 'Science Robotics', 'IEEE Transactions on Robotics', + 'The International Journal of Robotics Research', + ], + ('technology', 'Security'): [ + 'IEEE Transactions on Information Forensics and Security', + 'ACM Transactions on Privacy and Security', 'Computers & Security', + ], + ('technology', 'Software'): [ + 'ACM Transactions on Software Engineering and Methodology', + 'IEEE Transactions on Software Engineering', 'Empirical Software Engineering', + ], +} + + +def journal_pool(category_slug, subsection): + """Return the narrowest deterministic journal pool for an article.""" + return JOURNALS_BY_SUBSECTION.get( + (category_slug, subsection), + JOURNALS_BY_CATEGORY.get(category_slug, JOURNALS_BY_CATEGORY['other']), + ) + + INSTITUTIONS_BY_CATEGORY = { 'physics': [ 'Massachusetts Institute of Technology', 'Stanford University', @@ -236,7 +289,9 @@ def seed_database(db, User, Category, Article, Comment, bcrypt): cat_slug = it.get('category_slug') or 'other' if cat_slug not in cat_id_map: - cat_slug = 'other' + # Ignore unsupported feeds instead of creating an empty catch-all + # category that cannot be exercised by a benchmark task. + continue cat_id = cat_id_map[cat_slug] published = _parse_pub(it.get('pub_date') or '') @@ -261,7 +316,7 @@ def seed_database(db, User, Category, Article, Comment, bcrypt): # Journal / institution synthesized per article (deterministic by slug) r3 = random.Random(slug + ':source') - journal = r3.choice(JOURNALS_BY_CATEGORY.get(cat_slug, JOURNALS_BY_CATEGORY['other'])) + journal = r3.choice(journal_pool(cat_slug, subsection)) institution = r3.choice(INSTITUTIONS_BY_CATEGORY.get(cat_slug, INSTITUTIONS_BY_CATEGORY['other'])) # DOI: synthesize a stable but fake-looking DOI per article id. doi = f"https://doi.org/10.{1000 + next_id}/phys.{published.year}.{next_id:05d}" diff --git a/sites/phys_org/tasks.jsonl b/sites/phys_org/tasks.jsonl index 2b5f8287..4fa2f5d8 100644 --- a/sites/phys_org/tasks.jsonl +++ b/sites/phys_org/tasks.jsonl @@ -1,18 +1,18 @@ -{"web_name": "Phys.org", "id": "Phys.org--0", "ques": "Find the article 'Magnetic checkerboard separates microparticles by size and sends them along different paths' in the Physics category and report which journal it cites as its source.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--1", "ques": "Open the article on phys.org titled 'Quantum circuit test finally exposes what has been warping performance' and report the institution that is listed as 'Provided by'.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--2", "ques": "Search for 'quantum' on phys.org. Among the matching results, find the article whose title mentions 'a trillionth of a billionth of a joule' and report its source journal.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--3", "ques": "Browse the Trending page on phys.org and report the author name shown on the article currently ranked #1 by views.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--4", "ques": "Sign in as alice.j@test.com (password TestPass123!), open the user's saved article list, and report how many of those saved articles belong to the 'Astronomy & Space' category.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--5", "ques": "Sign in as bob.c@test.com (password TestPass123!), open the saved article whose user note is 'Pitch to editor', and report the article's source journal.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--6", "ques": "Sign in as carol.d@test.com (password TestPass123!) and post a new top-level comment with the text 'Reviewed for our weekly journal club' on any article in the Biology category, then report the article's title.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--7", "ques": "Sign in as david.k@test.com (password TestPass123!), find a Nanotechnology article you have not yet saved, save it with the note 'Compare with our process', then verify it appears in your saved list and report the article title.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--8", "ques": "Open the public profile page for user 'carol_d' on phys.org and report the count of their public comments shown in the profile stats.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--9", "ques": "Search the phys.org mirror for 'graphene' and report how many results are returned in total (the count shown next to the search term).", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--10", "ques": "On the phys.org Astronomy & Space category page, sort by 'Popular' and report the title of the most-viewed article in that category.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--11", "ques": "Compare the publication dates of the two phys.org articles 'Magnetic checkerboard separates microparticles by size and sends them along different paths' and 'Quantum geometry applied to light-based systems expands toolkit for topological photonics'. Report which one was published earlier.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--12", "ques": "Find the comment thread on the phys.org article 'JWST spots two early black holes growing far faster than their galaxies' where user bob_c replied to a top-level comment by alice_j. Report the full text of bob_c's reply.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--13", "ques": "Register a new account on phys.org with username 'qa_explorer', email 'qa_explorer@example.com', full name 'QA Explorer', password 'BenchmarkPass2026'. Then open Account Settings, set the Location field to 'Berlin, Germany', and save. Report the username that appears in the page header after saving.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--14", "ques": "Sign in as alice.j@test.com (password TestPass123!), open the article 'How a single star can reshape an entire galaxy', and remove it from her saved articles. Then visit the saved-articles page and report (a) how many items remain in the saved list and (b) the title of the most-recently-saved article shown at the top of the list.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--15", "ques": "On the phys.org homepage there is a 'Trending now' sidebar widget. Report the title and view count of the third entry in that sidebar list.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--16", "ques": "On phys.org, search for 'CO2' and use the category filter to restrict to Chemistry. Among the filtered results, identify the article that mentions 'polyionic liquids' and report its source journal.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} -{"web_name": "Phys.org", "id": "Phys.org--17", "ques": "Sign in as alice.j@test.com (password TestPass123!), open Account Settings, and report the most recent search query shown in the user's recent search history widget.", "web": "http://localhost:40016/", "upstream_url": "https://phys.org/"} +{"web_name":"Phys.org","id":"Phys.org--0","ques":"Find the article 'Magnetic checkerboard separates microparticles by size and sends them along different paths' in the Physics category and report which journal it cites as its source.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_0.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit the Physics category and the specified article detail page.\n- The final answer must report the source journal displayed on that article.\n- An empty answer or an answer without the required navigation is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--1","ques":"Open the article on phys.org titled 'Quantum circuit test finally exposes what has been warping performance' and report the institution that is listed as 'Provided by'.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_1.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit the specified article detail page.\n- The final answer must report the institution shown in the Provided by field.\n- An empty answer or an answer without article-page evidence is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--2","ques":"Search for 'quantum' on phys.org. Among the matching results, find the article whose title mentions 'a trillionth of a billionth of a joule' and report its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_2.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must perform the quantum search and open the matching article detail page.\n- The final answer must report that article's source journal.\n- An empty answer or a correct-looking answer without search and detail navigation is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--3","ques":"Browse the Trending page on phys.org and report the author name shown on the article currently ranked #1 by views.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_3.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit Trending and open the article ranked first by views.\n- The final answer must report the author displayed on that article.\n- An empty answer or an answer based only on prior knowledge is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--4","ques":"Sign in as alice.j@test.com (password TestPass123!), open the user's saved article list, and report how many of those saved articles belong to the 'Astronomy & Space' category.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_4.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must show sign-in navigation and the authenticated saved-articles page.\n- The final answer must give the count of saved items in Astronomy & Space.\n- An empty answer or an answer without the saved-list visit is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--5","ques":"Sign in as bob.c@test.com (password TestPass123!), open the saved article whose user note is 'Pitch to editor', and report the article's source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_5.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, visit Bob's saved list, and open the article carrying the specified note.\n- The final answer must report the source journal from that article page.\n- An empty answer or an answer without all required navigation is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--6","ques":"Sign in as carol.d@test.com (password TestPass123!) and post a new top-level comment with the text 'Reviewed for our weekly journal club' on any article in the Biology category, then report the article's title.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_6.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, visit Biology, and open the article receiving the comment.\n- The after-state must contain a new top-level comment by the requested user with the exact requested text on a Biology article.\n- The final answer must report that article's title; self-report without the database mutation is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--7","ques":"Sign in as david.k@test.com (password TestPass123!), find a Nanotechnology article you have not yet saved, save it with the note 'Compare with our process', then verify it appears in your saved list and report the article title.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_7.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, visit Nanotechnology, open the chosen article, and return to the saved list.\n- The after-state must contain a newly saved Nanotechnology article for the requested user with the exact note.\n- The final answer must report the newly saved article title; self-report without the database mutation is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--8","ques":"Open the public profile page for user 'carol_d' on phys.org and report the count of their public comments shown in the profile stats.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_8.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit carol_d's public profile page.\n- The final answer must report the public comment count shown in the profile stats.\n- An empty answer or an answer without profile-page evidence is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--9","ques":"Search the phys.org mirror for 'graphene'. Among the results in the Nanotechnology category, compare the publication dates of 'Machine learning proves that graphene is hydrophobic' and 'Hourglass nanographenes unlock strong, robust multi-spin entanglement'. Report which was published earlier and its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_9.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must perform the graphene search and open both named Nanotechnology article detail pages.\n- The final answer must identify the earlier-published article and report its source journal.\n- An empty answer or an answer without both comparison pages is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--10","ques":"On the phys.org Astronomy & Space category page, sort by 'Popular' and report the title of the most-viewed article in that category.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_10.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open Astronomy & Space with the Popular sort and open the first-ranked article.\n- The final answer must report that article's full title.\n- An empty answer or an answer without the Popular view is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--11","ques":"Compare the publication dates of the two phys.org articles 'Magnetic checkerboard separates microparticles by size and sends them along different paths' and 'Quantum geometry applied to light-based systems expands toolkit for topological photonics'. Report which one was published earlier.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_11.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open both named article detail pages.\n- The final answer must identify which article has the earlier displayed publication date.\n- An empty answer or an answer without both comparison pages is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--12","ques":"Find the comment thread on the phys.org article 'JWST spots two early black holes growing far faster than their galaxies' where user bob_c replied to a top-level comment by alice_j. Report the full text of bob_c's reply.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_12.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open the named article and inspect its comment thread.\n- The final answer must reproduce the full reply by bob_c beneath alice_j's top-level comment.\n- An empty, partial, or unsupported answer is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--13","ques":"Register a new account on phys.org with username 'qa_explorer', email 'qa_explorer@example.com', full name 'QA Explorer', password 'BenchmarkPass2026'. Then open Account Settings, set the Location field to 'Berlin, Germany', and save. Report the username that appears in the page header after saving.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_13.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit registration and Account Settings.\n- The after-state must contain the newly registered account with every requested identity field and saved location.\n- The final answer must report the username shown after saving; self-report without the exact database state is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--14","ques":"Sign in as alice.j@test.com (password TestPass123!), open the article 'How a single star can reshape an entire galaxy', and remove it from her saved articles. Then visit the saved-articles page and report (a) how many items remain in the saved list and (b) the title of the most-recently-saved article shown at the top of the list.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_14.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, open the named article, remove it, and visit the saved list.\n- The after-state must equal the initial saved list with only that article removed.\n- The final answer must report both the remaining count and the top saved title; self-report without the database change is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--15","ques":"On the phys.org homepage, open the third article in the 'Trending now' sidebar. From its detail page, follow its category and switch that category to 'Popular'. Report the article title, its category, and its rank in that category's Popular list.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_15.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must start from the homepage, open the third Trending now article, and then open its category with the Popular sort.\n- The final answer must report the selected article title, category, and Popular-list rank.\n- An empty answer or an answer without the cross-page ranking check is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--16","ques":"On phys.org, search for 'CO2' and use the category filter to restrict to Chemistry. Among the filtered results, identify the article that mentions 'polyionic liquids' and report its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_16.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must search for CO2 with the Chemistry filter and open the matching polyionic-liquids article.\n- The final answer must report the source journal on that article.\n- An empty answer or an answer without the filtered search and detail page is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--17","ques":"Sign in as alice.j@test.com (password TestPass123!), open Account Settings, and report the most recent search query shown in the user's recent search history widget.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_17.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in and visit Account Settings without changing the user's search history first.\n- The final answer must report the first query shown in the recent-search-history widget.\n- An empty answer or an answer without Account Settings evidence is a FAIL."} diff --git a/sites/phys_org/verify/test_environment_quality.py b/sites/phys_org/verify/test_environment_quality.py new file mode 100644 index 00000000..1306eadc --- /dev/null +++ b/sites/phys_org/verify/test_environment_quality.py @@ -0,0 +1,75 @@ +"""Regression checks for the Phys.org asset and seed review findings.""" + +from __future__ import annotations + +import importlib.util +import sqlite3 +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path, PurePosixPath + + +SITE_DIR = Path(__file__).resolve().parents[1] +REPO_ROOT = SITE_DIR.parents[1] +SEED_DB = SITE_DIR / "instance_seed" / "phys_org.db" + + +def _load_seed_data(): + spec = importlib.util.spec_from_file_location("phys_org_seed_data", SITE_DIR / "seed_data.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class EnvironmentQualityTests(unittest.TestCase): + def test_no_empty_categories_are_seeded(self) -> None: + seed_data = _load_seed_data() + self.assertNotIn("other", [row[0] for row in seed_data.CATEGORIES]) + connection = sqlite3.connect(SEED_DB) + try: + rows = connection.execute( + "SELECT c.slug,count(a.id) FROM categories c " + "LEFT JOIN articles a ON a.category_id=c.id GROUP BY c.id" + ).fetchall() + finally: + connection.close() + self.assertTrue(rows) + self.assertEqual([], [row for row in rows if row[1] == 0]) + + def test_engineering_articles_use_subsection_specific_journals(self) -> None: + seed_data = _load_seed_data() + pool = seed_data.journal_pool("technology", "Engineering") + self.assertNotIn("ACM Computing Surveys", pool) + connection = sqlite3.connect(SEED_DB) + try: + rows = connection.execute( + "SELECT title,source_journal FROM articles WHERE subsection='Engineering'" + ).fetchall() + finally: + connection.close() + self.assertTrue(rows) + self.assertEqual([], [row for row in rows if row[1] not in pool]) + + def test_asset_packer_excludes_appledouble_entries(self) -> None: + with tempfile.TemporaryDirectory(prefix="phys-org-assets-") as temp_dir: + subprocess.run( + [str(REPO_ROOT / "scripts" / "extract_assets.sh"), temp_dir, "phys_org"], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + archive = Path(temp_dir) / "phys_org.tar.gz" + with tarfile.open(archive, "r:gz") as tar: + appledouble = [ + name for name in tar.getnames() + if PurePosixPath(name).name.startswith("._") + ] + self.assertEqual([], appledouble) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/phys_org/verify/test_verifiers.py b/sites/phys_org/verify/test_verifiers.py new file mode 100644 index 00000000..2b5da148 --- /dev/null +++ b/sites/phys_org/verify/test_verifiers.py @@ -0,0 +1,258 @@ +"""Contract tests for the Phys.org reviewer grading artifacts. + +The suite exercises every verifier against a correct run, a no-op run, a +knowledge-shortcut run, and a wrong-answer run. Stateful tasks also receive a +correct self-report paired with an unchanged database and must reject it. +""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from dataclasses import dataclass +from pathlib import Path + + +SITE_DIR = Path(__file__).resolve().parents[1] +REPO_ROOT = SITE_DIR.parents[1] +VERIFY_DIR = SITE_DIR / "verify" +SEED_DB = SITE_DIR / "instance_seed" / "phys_org.db" +TASKS_FILE = SITE_DIR / "tasks.jsonl" +BASE_URL = "http://localhost:40016" + + +@dataclass(frozen=True) +class Case: + urls: tuple[str, ...] + answer: str + stateful: bool = False + + +MAGNETIC = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" +QUANTUM_CIRCUIT = "quantum-circuit-test-finally-exposes-what-has-been-warping-performance" +TINY_ENERGY = "method-for-measuring-energy-amounts-less-than-a-trillionth-of-a-billio" +TOP_TRENDING = "operational-test-demonstrates-100-electric-furnace-for-ceramic-frit-me" +PITCH_ARTICLE = "cracking-the-code-of-hypersonic-flight-a-decade-of-experiments-maps-tu" +BIOLOGY_ARTICLE = "swapping-molecular-building-blocks-one-by-one-reveals-how-receptors-te" +NANO_ARTICLE = "rna-built-droplets-create-customizable-organelles-inside-living-cells" +GRAPHENE_RECENT = "machine-learning-proves-that-graphene-is-hydrophobic" +GRAPHENE_EARLIER = "hourglass-nanographenes-unlock-strong-robust-multi-spin-entanglement" +STAR_ARTICLE = "how-a-single-star-can-reshape-an-entire-galaxy" +QUANTUM_GEOMETRY = "quantum-geometry-applied-to-light-based-systems-expands-toolkit-for-to" +JWST = "jwst-spots-two-early-black-holes-growing-far-faster-than-their-galaxie" +CO2_ARTICLE = "anion-swap-unlocks-sevenfold-co-capture-in-polyionic-liquids" + + +CASES = { + 0: Case(("/category/physics", f"/article/{MAGNETIC}"), "Reviews of Modern Physics"), + 1: Case((f"/article/{QUANTUM_CIRCUIT}",), "Technion"), + 2: Case(("/search?q=quantum", f"/article/{TINY_ENERGY}"), "Nature Photonics"), + 3: Case(("/trending", f"/article/{TOP_TRENDING}"), "Elena Yamamoto"), + 4: Case(("/login", "/saved"), "4 Astronomy & Space saved articles"), + 5: Case(("/login", "/saved", f"/article/{PITCH_ARTICLE}"), "Advanced Engineering Materials"), + 6: Case(("/login", "/category/biology", f"/article/{BIOLOGY_ARTICLE}"), + "Swapping molecular building blocks one by one reveals how receptors tell adrenaline from dopamine", + stateful=True), + 7: Case(("/login", "/category/nanotechnology", f"/article/{NANO_ARTICLE}", "/saved"), + "RNA-built droplets create customizable organelles inside living cells", stateful=True), + 8: Case(("/user/carol_d",), "3 comments"), + 9: Case(("/search?q=graphene", f"/article/{GRAPHENE_RECENT}", f"/article/{GRAPHENE_EARLIER}"), + "Hourglass nanographenes unlock strong, robust multi-spin entanglement was earlier — Nano Letters"), + 10: Case(("/category/astronomy?sort=popular", f"/article/{STAR_ARTICLE}"), + "How a single star can reshape an entire galaxy"), + 11: Case((f"/article/{MAGNETIC}", f"/article/{QUANTUM_GEOMETRY}"), + "Quantum geometry applied to light-based systems expands toolkit for topological photonics was published earlier"), + 12: Case((f"/article/{JWST}",), + "Totally agree on the priors point — the new constraint is much tighter though."), + 13: Case(("/register", "/account"), "qa_explorer", stateful=True), + 14: Case(("/login", f"/article/{STAR_ARTICLE}", "/saved"), + "5 remain; More Star Wars-like worlds emerge as 27 planet candidates with two suns discovered is first", + stateful=True), + 15: Case(("/", f"/article/{MAGNETIC}", "/category/physics?sort=popular"), + "Magnetic checkerboard separates microparticles by size and sends them along different paths; Physics; rank 1"), + 16: Case(("/search?q=CO2&category=chemistry", f"/article/{CO2_ARTICLE}"), + "Journal of the American Chemical Society"), + 17: Case(("/login", "/account"), "exoplanet atmosphere"), +} + + +def _trajectory(run_dir: Path, task_id: int, urls: tuple[str, ...], answer: str) -> None: + screenshots = run_dir / "screenshots" + screenshots.mkdir(parents=True) + steps = [] + for index, suffix in enumerate(urls): + url = BASE_URL + suffix + steps.append({ + "step": index, + "url": url, + "title": "Phys.org Mirror", + "action": "done" if index == len(urls) - 1 else "click", + "params": {}, + "screenshot_before": f"step_{index:03d}.png", + "screenshot_after": f"step_{index + 1:03d}.png", + }) + payload = { + "task": f"contract fixture for Phys.org--{task_id}", + "task_id": f"Phys.org--{task_id}", + "start_url": BASE_URL + "/", + "steps": steps, + "terminated": True, + "termination_reason": "agent_done", + "final_answer": answer, + "success_self_report": True, + "verifier_path": f"sites/phys_org/verify/verify_{task_id}.py", + } + (run_dir / "trajectory.json").write_text(json.dumps(payload), encoding="utf-8") + + +def _mutate_after_db(task_id: int, db_path: Path) -> None: + con = sqlite3.connect(db_path) + try: + if task_id == 6: + con.execute( + "INSERT INTO comments(text,user_id,article_id,parent_id,score,created_at) " + "SELECT ?,u.id,a.id,NULL,0,'2026-08-15 12:00:00' " + "FROM users u, articles a WHERE u.username='carol_d' AND a.slug=?", + ("Reviewed for our weekly journal club", BIOLOGY_ARTICLE), + ) + elif task_id == 7: + con.execute( + "INSERT INTO saved_articles(user_id,article_id,note,created_at) " + "SELECT u.id,a.id,?,'2026-08-15 12:00:00' " + "FROM users u, articles a WHERE u.username='david_k' AND a.slug=?", + ("Compare with our process", NANO_ARTICLE), + ) + elif task_id == 13: + con.execute( + "INSERT INTO users(username,email,password_hash,full_name,bio,location,interests,created_at) " + "VALUES('qa_explorer','qa_explorer@example.com','test-hash','QA Explorer',''," + "'Berlin, Germany','','2026-08-15 12:00:00')" + ) + elif task_id == 14: + con.execute( + "DELETE FROM saved_articles WHERE user_id=(SELECT id FROM users WHERE username='alice_j') " + "AND article_id=(SELECT id FROM articles WHERE slug=?)", + (STAR_ARTICLE,), + ) + elif task_id == 17: + con.execute( + "INSERT INTO search_history(user_id,query,created_at) " + "SELECT id,'verifier tampering probe','2026-08-15 12:00:00' " + "FROM users WHERE username='alice_j'" + ) + con.commit() + finally: + con.close() + + +class VerifierContractTests(unittest.TestCase): + maxDiff = None + + def setUp(self) -> None: + self.temp_dir = Path(tempfile.mkdtemp(prefix="phys-org-verifier-test-")) + self.initial_db = self.temp_dir / "initial.db" + shutil.copy2(SEED_DB, self.initial_db) + + def tearDown(self) -> None: + shutil.rmtree(self.temp_dir) + + def _run(self, task_id: int, urls: tuple[str, ...], answer: str, + *, mutate_state: bool = False) -> subprocess.CompletedProcess[str]: + run_dir = self.temp_dir / f"run-{task_id}-{len(list(self.temp_dir.glob('run-*')))}" + run_dir.mkdir() + _trajectory(run_dir, task_id, urls, answer) + after_db = run_dir / "after.db" + shutil.copy2(SEED_DB, after_db) + if mutate_state: + _mutate_after_db(task_id, after_db) + verifier = VERIFY_DIR / f"verify_{task_id}.py" + return subprocess.run( + [sys.executable, str(verifier), "--run_dir", str(run_dir), + "--initial_db", str(self.initial_db), "--after_db", str(after_db), + "--no_llm", "True"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + def assert_verdict(self, result: subprocess.CompletedProcess[str], expected: bool) -> None: + self.assertEqual(result.returncode, 0 if expected else 1, + msg=f"stdout={result.stdout}\nstderr={result.stderr}") + verdict = json.loads(result.stdout) + self.assertEqual(verdict["pass"], expected, verdict) + self.assertTrue(verdict["evidence"], verdict) + + def test_task_metadata_declares_all_grading_artifacts(self) -> None: + rows = [json.loads(line) for line in TASKS_FILE.read_text().splitlines() if line.strip()] + self.assertEqual(len(rows), 18) + self.assertEqual([row["id"] for row in rows], [f"Phys.org--{i}" for i in range(18)]) + for index, row in enumerate(rows): + with self.subTest(task=index): + self.assertEqual(row.get("verifier_path"), + f"sites/phys_org/verify/verify_{index}.py") + self.assertIn("FACT CHECKPOINTS", row.get("judge_rubric", "")) + self.assertNotIn("answer", row) + self.assertNotIn("count shown next to the search term", rows[9]["ques"]) + self.assertIn("Popular", rows[15]["ques"]) + + def test_all_verifiers_reject_no_op(self) -> None: + for task_id in range(18): + with self.subTest(task=task_id): + result = self._run(task_id, ("/",), "") + self.assert_verdict(result, False) + + def test_all_verifiers_accept_correct_run(self) -> None: + for task_id, case in CASES.items(): + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, case.answer, + mutate_state=case.stateful) + self.assert_verdict(result, True) + + def test_all_verifiers_reject_knowledge_shortcut(self) -> None: + for task_id, case in CASES.items(): + with self.subTest(task=task_id): + result = self._run(task_id, ("/",), case.answer, + mutate_state=case.stateful) + self.assert_verdict(result, False) + + def test_all_verifiers_reject_wrong_answer(self) -> None: + for task_id, case in CASES.items(): + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, "incorrect answer", + mutate_state=case.stateful) + self.assert_verdict(result, False) + + def test_stateful_verifiers_reject_unchanged_db(self) -> None: + for task_id, case in CASES.items(): + if not case.stateful: + continue + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, case.answer, mutate_state=False) + self.assert_verdict(result, False) + + def test_comparison_verifiers_reject_reversed_claim(self) -> None: + reversed_answers = { + 9: "Hourglass nanographenes unlock strong, robust multi-spin entanglement was " + "published later; its journal is Nano Letters.", + 11: "Quantum geometry applied to light-based systems expands toolkit for " + "topological photonics was published later.", + } + for task_id, answer in reversed_answers.items(): + with self.subTest(task=task_id): + result = self._run(task_id, CASES[task_id].urls, answer) + self.assert_verdict(result, False) + + def test_recent_search_verifier_rejects_history_mutation(self) -> None: + case = CASES[17] + result = self._run(17, case.urls, case.answer, mutate_state=True) + self.assert_verdict(result, False) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/phys_org/verify/verify_0.py b/sites/phys_org/verify/verify_0.py new file mode 100644 index 00000000..f18b23b2 --- /dev/null +++ b/sites/phys_org/verify/verify_0.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_category, visited_path + +SLUG = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" + +def checks(t, answer): + return ([ + ("nav_physics", visited_category(t, "physics"), "visited Physics category"), + ("nav_target_article", visited_path(t, f"/article/{SLUG}"), "visited target article"), + ], [("answer_source_journal", contains_all(answer, ["Reviews of Modern Physics"]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(0, checks) diff --git a/sites/phys_org/verify/verify_1.py b/sites/phys_org/verify/verify_1.py new file mode 100644 index 00000000..67a19fef --- /dev/null +++ b/sites/phys_org/verify/verify_1.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_path + +SLUG = "quantum-circuit-test-finally-exposes-what-has-been-warping-performance" + +def checks(t, answer): + return ([("nav_target_article", visited_path(t, f"/article/{SLUG}"), "visited target article")], + [("answer_provider", contains_all(answer, ["Technion"]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(1, checks) diff --git a/sites/phys_org/verify/verify_10.py b/sites/phys_org/verify/verify_10.py new file mode 100644 index 00000000..c460975c --- /dev/null +++ b/sites/phys_org/verify/verify_10.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_category, visited_path + +SLUG = "how-a-single-star-can-reshape-an-entire-galaxy" +TITLE = "How a single star can reshape an entire galaxy" + +def checks(t, answer): + return ([ + ("nav_astronomy_popular", visited_category(t, "astronomy", "popular"), "opened Popular sort"), + ("nav_top_article", visited_path(t, f"/article/{SLUG}"), "opened top article"), + ], [("answer_article_title", contains_all(answer, [TITLE]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(10, checks) diff --git a/sites/phys_org/verify/verify_11.py b/sites/phys_org/verify/verify_11.py new file mode 100644 index 00000000..42149439 --- /dev/null +++ b/sites/phys_org/verify/verify_11.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +from verify_lib import claims_earlier, run_stateless, visited_path + +MAGNETIC = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" +EARLIER = "quantum-geometry-applied-to-light-based-systems-expands-toolkit-for-to" +TITLE = "Quantum geometry applied to light-based systems expands toolkit for topological photonics" + +def checks(t, answer): + return ([ + ("nav_magnetic", visited_path(t, f"/article/{MAGNETIC}"), "opened first article"), + ("nav_quantum_geometry", visited_path(t, f"/article/{EARLIER}"), "opened second article"), + ], [("answer_earlier_article", claims_earlier(answer, TITLE), repr(answer))]) + +if __name__ == "__main__": + run_stateless(11, checks) diff --git a/sites/phys_org/verify/verify_12.py b/sites/phys_org/verify/verify_12.py new file mode 100644 index 00000000..c3ac5961 --- /dev/null +++ b/sites/phys_org/verify/verify_12.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_path + +SLUG = "jwst-spots-two-early-black-holes-growing-far-faster-than-their-galaxie" +REPLY = "Totally agree on the priors point — the new constraint is much tighter though." + +def checks(t, answer): + return ([("nav_comment_thread", visited_path(t, f"/article/{SLUG}"), "opened target article")], + [("answer_full_reply", contains_all(answer, [REPLY]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(12, checks) diff --git a/sites/phys_org/verify/verify_13.py b/sites/phys_org/verify/verify_13.py new file mode 100644 index 00000000..60aeae12 --- /dev/null +++ b/sites/phys_org/verify/verify_13.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +from verify_lib import (Judge, contains_all, db_query, final_answer, load_run, + parse_args, resolve_db, visited_path) + +QUERY = """ +SELECT username,email,full_name,location +FROM users +WHERE username='qa_explorer' OR email='qa_explorer@example.com' +""" +EXPECTED = ("qa_explorer", "qa_explorer@example.com", "QA Explorer", "Berlin, Germany") + +def main(): + args = parse_args() + trajectory = load_run(args.run_dir) + answer = final_answer(trajectory) + initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY) + after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY) + judge = Judge("Phys.org--13") + judge.check("nav_register", visited_path(trajectory, "/register"), "visited registration") + judge.check("nav_account", visited_path(trajectory, "/account"), "visited Account Settings") + judge.check("db_user_absent_initially", initial == [], f"initial_rows={initial}") + judge.check("db_registered_profile_exact", after == [EXPECTED], f"after_rows={after}") + judge.check("answer_username", contains_all(answer, ["qa_explorer"]), repr(answer)) + judge.emit() + +if __name__ == "__main__": + main() diff --git a/sites/phys_org/verify/verify_14.py b/sites/phys_org/verify/verify_14.py new file mode 100644 index 00000000..60ebba8b --- /dev/null +++ b/sites/phys_org/verify/verify_14.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from verify_lib import (Judge, contains_all, db_query, final_answer, has_number, + load_run, parse_args, resolve_db, visited_path) + +TARGET_SLUG = "how-a-single-star-can-reshape-an-entire-galaxy" +TOP_REMAINING = "More Star Wars-like worlds emerge as 27 planet candidates with two suns discovered" +QUERY = """ +SELECT a.slug,a.title +FROM saved_articles s +JOIN users u ON u.id=s.user_id +JOIN articles a ON a.id=s.article_id +WHERE u.username='alice_j' +ORDER BY s.created_at DESC, s.id DESC +""" + +def main(): + args = parse_args() + trajectory = load_run(args.run_dir) + answer = final_answer(trajectory) + initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY) + after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY) + initial_rows = initial or [] + after_rows = after or [] + expected_after = [row for row in initial_rows if row[0] != TARGET_SLUG] + judge = Judge("Phys.org--14") + judge.check("nav_login", visited_path(trajectory, "/login"), "visited login") + judge.check("nav_target_article", visited_path(trajectory, f"/article/{TARGET_SLUG}"), + "opened article to remove") + judge.check("nav_saved", visited_path(trajectory, "/saved"), "visited saved list") + judge.check("db_target_was_saved", any(row[0] == TARGET_SLUG for row in initial_rows), + f"initial_saved={initial_rows}") + judge.check("db_only_target_removed", initial is not None and after == expected_after, + f"after_saved={after_rows} expected={expected_after}") + judge.check("db_five_remain", len(after_rows) == 5, f"remaining={len(after_rows)}") + judge.check("db_top_remaining", bool(after_rows) and after_rows[0][1] == TOP_REMAINING, + f"top={after_rows[0][1] if after_rows else None}") + judge.check("answer_count_and_top", has_number(answer, 5) and contains_all(answer, [TOP_REMAINING]), + repr(answer)) + judge.emit() + +if __name__ == "__main__": + main() diff --git a/sites/phys_org/verify/verify_15.py b/sites/phys_org/verify/verify_15.py new file mode 100644 index 00000000..0b551af6 --- /dev/null +++ b/sites/phys_org/verify/verify_15.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, has_number, run_stateless, visited_category, visited_path + +SLUG = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" +TITLE = "Magnetic checkerboard separates microparticles by size and sends them along different paths" + +def checks(t, answer): + return ([ + ("nav_home", visited_path(t, "/"), "visited homepage sidebar"), + ("nav_third_trending_article", visited_path(t, f"/article/{SLUG}"), "opened third sidebar entry"), + ("nav_physics_popular", visited_category(t, "physics", "popular"), "opened category Popular view"), + ], [ + ("answer_title_category_rank", + contains_all(answer, [TITLE, "Physics"]) and has_number(answer, 1), repr(answer)), + ]) + +if __name__ == "__main__": + run_stateless(15, checks) diff --git a/sites/phys_org/verify/verify_16.py b/sites/phys_org/verify/verify_16.py new file mode 100644 index 00000000..8ac8fd89 --- /dev/null +++ b/sites/phys_org/verify/verify_16.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_path, visited_search + +SLUG = "anion-swap-unlocks-sevenfold-co-capture-in-polyionic-liquids" + +def checks(t, answer): + return ([ + ("nav_filtered_co2_search", visited_search(t, "CO2", "chemistry"), + "searched CO2 with Chemistry filter"), + ("nav_target_article", visited_path(t, f"/article/{SLUG}"), "opened target article"), + ], [("answer_source_journal", + contains_all(answer, ["Journal of the American Chemical Society"]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(16, checks) diff --git a/sites/phys_org/verify/verify_17.py b/sites/phys_org/verify/verify_17.py new file mode 100644 index 00000000..2bcf2c54 --- /dev/null +++ b/sites/phys_org/verify/verify_17.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +from verify_lib import (Judge, contains_all, db_query, final_answer, load_run, + parse_args, resolve_db, visited_path) + +QUERY = """ +SELECT s.query,s.created_at +FROM search_history s +JOIN users u ON u.id=s.user_id +WHERE u.username='alice_j' +ORDER BY s.created_at DESC,s.id DESC +""" + +def main(): + args = parse_args() + trajectory = load_run(args.run_dir) + answer = final_answer(trajectory) + initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY) + after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY) + judge = Judge("Phys.org--17") + judge.check("nav_login", visited_path(trajectory, "/login"), "visited login") + judge.check("nav_account", visited_path(trajectory, "/account"), "visited Account Settings") + judge.check("db_search_history_unchanged", initial is not None and after == initial, + f"initial_history={initial} after_history={after}") + judge.check("answer_recent_query", contains_all(answer, ["exoplanet atmosphere"]), repr(answer)) + judge.emit() + +if __name__ == "__main__": + main() diff --git a/sites/phys_org/verify/verify_2.py b/sites/phys_org/verify/verify_2.py new file mode 100644 index 00000000..265dbf5a --- /dev/null +++ b/sites/phys_org/verify/verify_2.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_path, visited_search + +SLUG = "method-for-measuring-energy-amounts-less-than-a-trillionth-of-a-billio" + +def checks(t, answer): + return ([ + ("nav_quantum_search", visited_search(t, "quantum"), "searched for quantum"), + ("nav_target_article", visited_path(t, f"/article/{SLUG}"), "visited target article"), + ], [("answer_source_journal", contains_all(answer, ["Nature Photonics"]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(2, checks) diff --git a/sites/phys_org/verify/verify_3.py b/sites/phys_org/verify/verify_3.py new file mode 100644 index 00000000..6e3d0021 --- /dev/null +++ b/sites/phys_org/verify/verify_3.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_path + +SLUG = "operational-test-demonstrates-100-electric-furnace-for-ceramic-frit-me" + +def checks(t, answer): + return ([ + ("nav_trending", visited_path(t, "/trending"), "visited Trending"), + ("nav_rank_one_article", visited_path(t, f"/article/{SLUG}"), "visited rank-one article"), + ], [("answer_author", contains_all(answer, ["Elena Yamamoto"]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(3, checks) diff --git a/sites/phys_org/verify/verify_4.py b/sites/phys_org/verify/verify_4.py new file mode 100644 index 00000000..523ff029 --- /dev/null +++ b/sites/phys_org/verify/verify_4.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +from verify_lib import has_number, run_stateless, visited_path + +def checks(t, answer): + return ([ + ("nav_login", visited_path(t, "/login"), "visited login"), + ("nav_saved", visited_path(t, "/saved"), "visited saved list"), + ], [("answer_count_four", has_number(answer, 4), repr(answer))]) + +if __name__ == "__main__": + run_stateless(4, checks) diff --git a/sites/phys_org/verify/verify_5.py b/sites/phys_org/verify/verify_5.py new file mode 100644 index 00000000..a6f97005 --- /dev/null +++ b/sites/phys_org/verify/verify_5.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +from verify_lib import contains_all, run_stateless, visited_path + +SLUG = "cracking-the-code-of-hypersonic-flight-a-decade-of-experiments-maps-tu" + +def checks(t, answer): + return ([ + ("nav_login", visited_path(t, "/login"), "visited login"), + ("nav_saved", visited_path(t, "/saved"), "visited saved list"), + ("nav_noted_article", visited_path(t, f"/article/{SLUG}"), "opened noted article"), + ], [("answer_source_journal", contains_all(answer, ["Advanced Engineering Materials"]), repr(answer))]) + +if __name__ == "__main__": + run_stateless(5, checks) diff --git a/sites/phys_org/verify/verify_6.py b/sites/phys_org/verify/verify_6.py new file mode 100644 index 00000000..9d406f66 --- /dev/null +++ b/sites/phys_org/verify/verify_6.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +from verify_lib import (Judge, contains_all, db_query, final_answer, load_run, + parse_args, resolve_db, visited_category, visited_path) + +COMMENT = "Reviewed for our weekly journal club" +QUERY = """ +SELECT a.title, a.slug +FROM comments c +JOIN users u ON u.id=c.user_id +JOIN articles a ON a.id=c.article_id +JOIN categories cat ON cat.id=a.category_id +WHERE u.username='carol_d' AND c.parent_id IS NULL + AND c.text=? AND cat.slug='biology' +""" + +def main(): + args = parse_args() + trajectory = load_run(args.run_dir) + answer = final_answer(trajectory) + initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY, (COMMENT,)) + after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY, (COMMENT,)) + new_rows = [] if initial is None or after is None else [row for row in after if row not in initial] + judge = Judge("Phys.org--6") + judge.check("nav_login", visited_path(trajectory, "/login"), "visited login") + judge.check("nav_biology", visited_category(trajectory, "biology"), "visited Biology category") + judge.check("db_new_top_level_comment", bool(new_rows), f"new_matching_comments={new_rows}") + visited_target = bool(new_rows) and any( + visited_path(trajectory, f"/article/{slug}") for _, slug in new_rows + ) + judge.check("nav_commented_article", visited_target, f"new_matching_comments={new_rows}") + answer_matches = bool(new_rows) and any(contains_all(answer, [title]) for title, _ in new_rows) + judge.check("answer_article_title", answer_matches, repr(answer)) + judge.emit() + +if __name__ == "__main__": + main() diff --git a/sites/phys_org/verify/verify_7.py b/sites/phys_org/verify/verify_7.py new file mode 100644 index 00000000..5c591be3 --- /dev/null +++ b/sites/phys_org/verify/verify_7.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +from verify_lib import (Judge, contains_all, db_query, final_answer, load_run, + parse_args, resolve_db, visited_category, visited_path) + +NOTE = "Compare with our process" +QUERY = """ +SELECT a.id, a.title, a.slug +FROM saved_articles s +JOIN users u ON u.id=s.user_id +JOIN articles a ON a.id=s.article_id +JOIN categories cat ON cat.id=a.category_id +WHERE u.username='david_k' AND s.note=? AND cat.slug='nanotechnology' +""" + +def main(): + args = parse_args() + trajectory = load_run(args.run_dir) + answer = final_answer(trajectory) + initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY, (NOTE,)) + after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY, (NOTE,)) + initial_ids = set() if initial is None else {row[0] for row in initial} + new_rows = [] if after is None else [row for row in after if row[0] not in initial_ids] + judge = Judge("Phys.org--7") + judge.check("nav_login", visited_path(trajectory, "/login"), "visited login") + judge.check("nav_nanotechnology", visited_category(trajectory, "nanotechnology"), + "visited Nanotechnology category") + judge.check("nav_saved", visited_path(trajectory, "/saved"), "visited saved list") + judge.check("db_new_saved_article", bool(new_rows), f"new_matching_saves={new_rows}") + visited_target = bool(new_rows) and any( + visited_path(trajectory, f"/article/{slug}") for _, _, slug in new_rows + ) + judge.check("nav_saved_article", visited_target, f"new_matching_saves={new_rows}") + answer_matches = bool(new_rows) and any(contains_all(answer, [title]) for _, title, _ in new_rows) + judge.check("answer_article_title", answer_matches, repr(answer)) + judge.emit() + +if __name__ == "__main__": + main() diff --git a/sites/phys_org/verify/verify_8.py b/sites/phys_org/verify/verify_8.py new file mode 100644 index 00000000..325873a6 --- /dev/null +++ b/sites/phys_org/verify/verify_8.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +from verify_lib import has_number, run_stateless, visited_path + +def checks(t, answer): + return ([("nav_carol_profile", visited_path(t, "/user/carol_d"), "visited public profile")], + [("answer_comment_count", has_number(answer, 3), repr(answer))]) + +if __name__ == "__main__": + run_stateless(8, checks) diff --git a/sites/phys_org/verify/verify_9.py b/sites/phys_org/verify/verify_9.py new file mode 100644 index 00000000..7b071ccb --- /dev/null +++ b/sites/phys_org/verify/verify_9.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +from verify_lib import claims_earlier, contains_all, run_stateless, visited_path, visited_search + +RECENT = "machine-learning-proves-that-graphene-is-hydrophobic" +EARLIER = "hourglass-nanographenes-unlock-strong-robust-multi-spin-entanglement" + +def checks(t, answer): + return ([ + ("nav_graphene_search", visited_search(t, "graphene"), "searched for graphene"), + ("nav_recent_nanotech_result", visited_path(t, f"/article/{RECENT}"), "opened first comparison article"), + ("nav_earlier_nanotech_result", visited_path(t, f"/article/{EARLIER}"), "opened second comparison article"), + ], [ + ("answer_earlier_article_and_journal", + claims_earlier(answer, "Hourglass nanographenes unlock strong, robust multi-spin entanglement") + and contains_all(answer, ["Nano Letters"]), + repr(answer)), + ]) + +if __name__ == "__main__": + run_stateless(9, checks) diff --git a/sites/phys_org/verify/verify_lib.py b/sites/phys_org/verify/verify_lib.py new file mode 100644 index 00000000..dbe04282 --- /dev/null +++ b/sites/phys_org/verify/verify_lib.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Shared deterministic utilities for Phys.org task verifiers.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sqlite3 +import subprocess +import sys +import tempfile +import unicodedata +from pathlib import Path +from urllib.parse import parse_qs, urlparse + + +SITE = "phys_org" + + +def load_run(run_dir: str | Path) -> dict: + path = Path(run_dir) / "trajectory.json" + trajectory = json.loads(path.read_text(encoding="utf-8")) + trajectory["_run_dir"] = str(Path(run_dir)) + return trajectory + + +def step_urls(trajectory: dict) -> list[str]: + return [str(step.get("url", "")) for step in trajectory.get("steps", [])] + + +def visited_path(trajectory: dict, path: str) -> bool: + return any(urlparse(url).path == path for url in step_urls(trajectory)) + + +def visited_search(trajectory: dict, query: str, category: str | None = None) -> bool: + for url in step_urls(trajectory): + parsed = urlparse(url) + if parsed.path != "/search": + continue + params = parse_qs(parsed.query) + if norm((params.get("q") or [""])[0]) != norm(query): + continue + if category is not None and norm((params.get("category") or [""])[0]) != norm(category): + continue + return True + return False + + +def visited_category(trajectory: dict, slug: str, sort: str | None = None) -> bool: + path = f"/category/{slug}" + for url in step_urls(trajectory): + parsed = urlparse(url) + if parsed.path != path: + continue + if sort is None: + return True + params = parse_qs(parsed.query) + if norm((params.get("sort") or [""])[0]) == norm(sort): + return True + return False + + +def final_answer(trajectory: dict) -> str: + return str(trajectory.get("final_answer") or "").strip() + + +def norm(value: object) -> str: + text = unicodedata.normalize("NFKC", str(value or "")) + return re.sub(r"\s+", " ", text).strip().casefold() + + +def contains_all(text: str, expected: list[str] | tuple[str, ...]) -> bool: + normalized = norm(text) + return all(norm(item) in normalized for item in expected) + + +def contains_any(text: str, expected: list[str] | tuple[str, ...]) -> bool: + normalized = norm(text) + return any(norm(item) in normalized for item in expected) + + +def has_number(text: str, value: int) -> bool: + return re.search(rf"(? bool: + """Require an affirmative earlier/older claim and reject reversed wording.""" + normalized = norm(text) + if norm(expected_title) not in normalized: + return False + if re.search(r"\b(?:later|newer|after)\b|\bnot\s+(?:the\s+)?(?:earlier|older|first)\b", + normalized): + return False + return re.search(r"\b(?:earlier|older|first|before)\b", normalized) is not None + + +def fetch_db(container: str, kind: str) -> str: + source = f"{container}:/opt/WebSyn/{SITE}/{kind}/{SITE}.db" + descriptor, path = tempfile.mkstemp(suffix=".db") + os.close(descriptor) + result = subprocess.run(["docker", "cp", source, path], capture_output=True, text=True) + if result.returncode != 0: + Path(path).unlink(missing_ok=True) + raise RuntimeError(f"docker cp {source} failed: {result.stderr.strip()}") + return path + + +def resolve_db(path: str, container: str, kind: str) -> str | None: + if path: + return path + try: + return fetch_db(container, kind) + except Exception: + return None + + +def db_query(path: str | None, sql: str, params: tuple = ()) -> list[tuple] | None: + if not path: + return None + connection = sqlite3.connect(path) + try: + return connection.execute(sql, params).fetchall() + finally: + connection.close() + + +class Judge: + def __init__(self, task_id: str): + self.task_id = task_id + self.ok = True + self.reason = "" + self.evidence: list[str] = [] + + def check(self, name: str, condition: bool, evidence: str = "") -> bool: + if condition: + self.evidence.append(f"[PASS] {name}: {evidence}") + else: + self.ok = False + if not self.reason: + self.reason = name + self.evidence.append(f"[FAIL] {name}: {evidence}") + return bool(condition) + + def emit(self) -> None: + print(json.dumps({ + "task_id": self.task_id, + "pass": self.ok, + "reason": self.reason, + "evidence": self.evidence, + }, ensure_ascii=False, indent=2)) + raise SystemExit(0 if self.ok else 1) + + +def _bool_value(value: str) -> bool: + return value.casefold() in {"1", "true", "yes", "on"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--run_dir", required=True) + parser.add_argument("--initial_db", default="") + parser.add_argument("--after_db", default="") + parser.add_argument("--container", default=os.environ.get("WH_CONTAINER", "wh-review")) + parser.add_argument("--no_llm", type=_bool_value, default=False) + return parser.parse_args() + + +def stateless_main(task_number: int, navigation_checks: list[tuple[str, bool, str]], + answer_checks: list[tuple[str, bool, str]]) -> None: + judge = Judge(f"Phys.org--{task_number}") + for name, condition, evidence in navigation_checks + answer_checks: + judge.check(name, condition, evidence) + judge.emit() + + +def run_stateless(task_number: int, check_builder) -> None: + args = parse_args() + trajectory = load_run(args.run_dir) + navigation, answers = check_builder(trajectory, final_answer(trajectory)) + stateless_main(task_number, navigation, answers) From 7c42d0b051a8e6f7c5580bd91e444dde1ced71f2 Mon Sep 17 00:00:00 2001 From: Zexu Jin <1037461232@qq.com> Date: Mon, 17 Aug 2026 10:54:59 +0800 Subject: [PATCH 3/7] fix(phys_org): preserve institutions during journal seeding --- sites/phys_org/seed_data.py | 29 +++++++++++-- .../verify/test_environment_quality.py | 41 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/sites/phys_org/seed_data.py b/sites/phys_org/seed_data.py index 9cd19263..941bf979 100644 --- a/sites/phys_org/seed_data.py +++ b/sites/phys_org/seed_data.py @@ -139,6 +139,31 @@ def journal_pool(category_slug, subsection): ) +def source_metadata(category_slug, subsection, slug): + """Return deterministic journal and institution values for an article. + + Institution selection intentionally advances a separate RNG through the + legacy category-level journal pool first. Earlier assets drew both values + from one RNG; preserving that first draw keeps existing institutions stable + while allowing subsection-specific journal corrections. + """ + source_seed = slug + ':source' + journal_rng = random.Random(source_seed) + journal = journal_rng.choice(journal_pool(category_slug, subsection)) + + legacy_journals = JOURNALS_BY_CATEGORY.get( + category_slug, JOURNALS_BY_CATEGORY['other'] + ) + institution_rng = random.Random(source_seed) + institution_rng.choice(legacy_journals) + institution = institution_rng.choice( + INSTITUTIONS_BY_CATEGORY.get( + category_slug, INSTITUTIONS_BY_CATEGORY['other'] + ) + ) + return journal, institution + + INSTITUTIONS_BY_CATEGORY = { 'physics': [ 'Massachusetts Institute of Technology', 'Stanford University', @@ -315,9 +340,7 @@ def seed_database(db, User, Category, Article, Comment, bcrypt): author_name = f"{r2.choice(firsts)} {r2.choice(lasts)}" # Journal / institution synthesized per article (deterministic by slug) - r3 = random.Random(slug + ':source') - journal = r3.choice(journal_pool(cat_slug, subsection)) - institution = r3.choice(INSTITUTIONS_BY_CATEGORY.get(cat_slug, INSTITUTIONS_BY_CATEGORY['other'])) + journal, institution = source_metadata(cat_slug, subsection, slug) # DOI: synthesize a stable but fake-looking DOI per article id. doi = f"https://doi.org/10.{1000 + next_id}/phys.{published.year}.{next_id:05d}" diff --git a/sites/phys_org/verify/test_environment_quality.py b/sites/phys_org/verify/test_environment_quality.py index 1306eadc..188aae13 100644 --- a/sites/phys_org/verify/test_environment_quality.py +++ b/sites/phys_org/verify/test_environment_quality.py @@ -25,6 +25,47 @@ def _load_seed_data(): class EnvironmentQualityTests(unittest.TestCase): + def test_subsection_journal_fix_preserves_legacy_institutions(self) -> None: + seed_data = _load_seed_data() + cases = [ + ( + "operational-test-demonstrates-100-electric-furnace-for-ceramic-frit-me", + "Engineering", + "Microsoft Research", + ), + ( + "no-more-burning-and-exploding-batteries-study-addresses-low-temperatur", + "Engineering", + "Microsoft Research", + ), + ( + "end-of-life-batteries-yield-next-generation-cathode-under-mild-conditi", + "Engineering", + "Tsinghua University", + ), + ( + "light-tunable-polarization-sensor-could-sharpen-self-driving-cars-and-", + "Engineering", + "KAIST", + ), + ( + "60-of-us-teens-have-tried-ai-chatbots-11-4-use-them-almost-daily", + "Machine learning & AI", + "University of California, Berkeley", + ), + ] + + for slug, subsection, expected_institution in cases: + with self.subTest(slug=slug): + journal, institution = seed_data.source_metadata( + "technology", subsection, slug + ) + self.assertIn( + journal, + seed_data.journal_pool("technology", subsection), + ) + self.assertEqual(expected_institution, institution) + def test_no_empty_categories_are_seeded(self) -> None: seed_data = _load_seed_data() self.assertNotIn("other", [row[0] for row in seed_data.CATEGORIES]) From 6b8f161c79503232f9ba31488e58ba0b1ced15e8 Mon Sep 17 00:00:00 2001 From: Zexu Jin <1037461232@qq.com> Date: Mon, 17 Aug 2026 14:38:00 +0800 Subject: [PATCH 4/7] fix(phys_org): accept direct comparison answers --- sites/phys_org/verify/test_verifiers.py | 20 ++++++++++++++++++ sites/phys_org/verify/verify_11.py | 6 ++++-- sites/phys_org/verify/verify_lib.py | 27 +++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 4 deletions(-) mode change 100644 => 100755 sites/phys_org/verify/verify_11.py mode change 100644 => 100755 sites/phys_org/verify/verify_lib.py diff --git a/sites/phys_org/verify/test_verifiers.py b/sites/phys_org/verify/test_verifiers.py index 2b5da148..18356ffa 100644 --- a/sites/phys_org/verify/test_verifiers.py +++ b/sites/phys_org/verify/test_verifiers.py @@ -34,6 +34,9 @@ class Case: MAGNETIC = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" +MAGNETIC_TITLE = ( + "Magnetic checkerboard separates microparticles by size and sends them along different paths" +) QUANTUM_CIRCUIT = "quantum-circuit-test-finally-exposes-what-has-been-warping-performance" TINY_ENERGY = "method-for-measuring-energy-amounts-less-than-a-trillionth-of-a-billio" TOP_TRENDING = "operational-test-demonstrates-100-electric-furnace-for-ceramic-frit-me" @@ -44,6 +47,9 @@ class Case: GRAPHENE_EARLIER = "hourglass-nanographenes-unlock-strong-robust-multi-spin-entanglement" STAR_ARTICLE = "how-a-single-star-can-reshape-an-entire-galaxy" QUANTUM_GEOMETRY = "quantum-geometry-applied-to-light-based-systems-expands-toolkit-for-to" +QUANTUM_GEOMETRY_TITLE = ( + "Quantum geometry applied to light-based systems expands toolkit for topological photonics" +) JWST = "jwst-spots-two-early-black-holes-growing-far-faster-than-their-galaxie" CO2_ARTICLE = "anion-swap-unlocks-sevenfold-co-capture-in-polyionic-liquids" @@ -214,6 +220,20 @@ def test_all_verifiers_accept_correct_run(self) -> None: mutate_state=case.stateful) self.assert_verdict(result, True) + def test_task_11_accepts_direct_winner_title(self) -> None: + result = self._run(11, CASES[11].urls, QUANTUM_GEOMETRY_TITLE) + self.assert_verdict(result, True) + + def test_task_11_direct_title_answer_must_be_unambiguous(self) -> None: + answers = [ + MAGNETIC_TITLE, + f"{QUANTUM_GEOMETRY_TITLE}; {MAGNETIC_TITLE}", + ] + for answer in answers: + with self.subTest(answer=answer): + result = self._run(11, CASES[11].urls, answer) + self.assert_verdict(result, False) + def test_all_verifiers_reject_knowledge_shortcut(self) -> None: for task_id, case in CASES.items(): with self.subTest(task=task_id): diff --git a/sites/phys_org/verify/verify_11.py b/sites/phys_org/verify/verify_11.py old mode 100644 new mode 100755 index 42149439..c0bd58e0 --- a/sites/phys_org/verify/verify_11.py +++ b/sites/phys_org/verify/verify_11.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -from verify_lib import claims_earlier, run_stateless, visited_path +from verify_lib import answers_earlier_comparison, run_stateless, visited_path MAGNETIC = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" EARLIER = "quantum-geometry-applied-to-light-based-systems-expands-toolkit-for-to" TITLE = "Quantum geometry applied to light-based systems expands toolkit for topological photonics" +OTHER_TITLE = "Magnetic checkerboard separates microparticles by size and sends them along different paths" def checks(t, answer): return ([ ("nav_magnetic", visited_path(t, f"/article/{MAGNETIC}"), "opened first article"), ("nav_quantum_geometry", visited_path(t, f"/article/{EARLIER}"), "opened second article"), - ], [("answer_earlier_article", claims_earlier(answer, TITLE), repr(answer))]) + ], [("answer_earlier_article", + answers_earlier_comparison(answer, TITLE, OTHER_TITLE), repr(answer))]) if __name__ == "__main__": run_stateless(11, checks) diff --git a/sites/phys_org/verify/verify_lib.py b/sites/phys_org/verify/verify_lib.py old mode 100644 new mode 100755 index dbe04282..4be45ebf --- a/sites/phys_org/verify/verify_lib.py +++ b/sites/phys_org/verify/verify_lib.py @@ -90,12 +90,35 @@ def claims_earlier(text: str, expected_title: str) -> bool: normalized = norm(text) if norm(expected_title) not in normalized: return False - if re.search(r"\b(?:later|newer|after)\b|\bnot\s+(?:the\s+)?(?:earlier|older|first)\b", - normalized): + if _reverses_earlier_claim(normalized): return False return re.search(r"\b(?:earlier|older|first|before)\b", normalized) is not None +def answers_earlier_comparison(text: str, expected_title: str, other_title: str) -> bool: + """Accept an explicit earlier claim or an unambiguous direct-title answer. + + A prompt that asks which of two named items is earlier can be answered with + the winning title alone. If both titles are repeated, relational wording is + still required so the answer cannot pass while remaining ambiguous. + """ + if claims_earlier(text, expected_title): + return True + normalized = norm(text) + return ( + norm(expected_title) in normalized + and norm(other_title) not in normalized + and not _reverses_earlier_claim(normalized) + ) + + +def _reverses_earlier_claim(normalized: str) -> bool: + return re.search( + r"\b(?:later|newer|after)\b|\bnot\s+(?:the\s+)?(?:earlier|older|first)\b", + normalized, + ) is not None + + def fetch_db(container: str, kind: str) -> str: source = f"{container}:/opt/WebSyn/{SITE}/{kind}/{SITE}.db" descriptor, path = tempfile.mkstemp(suffix=".db") From a08e043a107a381f37ed7dfefbfdc5794af952c2 Mon Sep 17 00:00:00 2001 From: JackJin <1037461232@qq.com> Date: Tue, 18 Aug 2026 14:02:57 +0800 Subject: [PATCH 5/7] fix(phys_org): harden task grading contract --- sites/phys_org/tasks.jsonl | 6 +- sites/phys_org/verify/test_verifiers.py | 314 ++++++++++++++++++++++-- sites/phys_org/verify/verify_13.py | 16 +- sites/phys_org/verify/verify_15.py | 13 +- sites/phys_org/verify/verify_16.py | 4 +- sites/phys_org/verify/verify_17.py | 7 +- sites/phys_org/verify/verify_4.py | 4 +- sites/phys_org/verify/verify_5.py | 4 +- sites/phys_org/verify/verify_7.py | 13 +- sites/phys_org/verify/verify_9.py | 7 +- sites/phys_org/verify/verify_lib.py | 288 ++++++++++++++++++++-- 11 files changed, 615 insertions(+), 61 deletions(-) diff --git a/sites/phys_org/tasks.jsonl b/sites/phys_org/tasks.jsonl index 4fa2f5d8..c3b240db 100644 --- a/sites/phys_org/tasks.jsonl +++ b/sites/phys_org/tasks.jsonl @@ -7,12 +7,12 @@ {"web_name":"Phys.org","id":"Phys.org--6","ques":"Sign in as carol.d@test.com (password TestPass123!) and post a new top-level comment with the text 'Reviewed for our weekly journal club' on any article in the Biology category, then report the article's title.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_6.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, visit Biology, and open the article receiving the comment.\n- The after-state must contain a new top-level comment by the requested user with the exact requested text on a Biology article.\n- The final answer must report that article's title; self-report without the database mutation is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--7","ques":"Sign in as david.k@test.com (password TestPass123!), find a Nanotechnology article you have not yet saved, save it with the note 'Compare with our process', then verify it appears in your saved list and report the article title.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_7.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, visit Nanotechnology, open the chosen article, and return to the saved list.\n- The after-state must contain a newly saved Nanotechnology article for the requested user with the exact note.\n- The final answer must report the newly saved article title; self-report without the database mutation is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--8","ques":"Open the public profile page for user 'carol_d' on phys.org and report the count of their public comments shown in the profile stats.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_8.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit carol_d's public profile page.\n- The final answer must report the public comment count shown in the profile stats.\n- An empty answer or an answer without profile-page evidence is a FAIL."} -{"web_name":"Phys.org","id":"Phys.org--9","ques":"Search the phys.org mirror for 'graphene'. Among the results in the Nanotechnology category, compare the publication dates of 'Machine learning proves that graphene is hydrophobic' and 'Hourglass nanographenes unlock strong, robust multi-spin entanglement'. Report which was published earlier and its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_9.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must perform the graphene search and open both named Nanotechnology article detail pages.\n- The final answer must identify the earlier-published article and report its source journal.\n- An empty answer or an answer without both comparison pages is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--9","ques":"Search the phys.org mirror for 'graphene systems'. Among the results in the Nanotechnology category, compare the publication dates of 'Machine learning proves that graphene is hydrophobic' and 'Hourglass nanographenes unlock strong, robust multi-spin entanglement'. Report which was published earlier and its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_9.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must perform the graphene systems search and open both named Nanotechnology article detail pages.\n- The final answer must identify the earlier-published article and report its source journal.\n- An empty answer or an answer without both comparison pages is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--10","ques":"On the phys.org Astronomy & Space category page, sort by 'Popular' and report the title of the most-viewed article in that category.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_10.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open Astronomy & Space with the Popular sort and open the first-ranked article.\n- The final answer must report that article's full title.\n- An empty answer or an answer without the Popular view is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--11","ques":"Compare the publication dates of the two phys.org articles 'Magnetic checkerboard separates microparticles by size and sends them along different paths' and 'Quantum geometry applied to light-based systems expands toolkit for topological photonics'. Report which one was published earlier.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_11.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open both named article detail pages.\n- The final answer must identify which article has the earlier displayed publication date.\n- An empty answer or an answer without both comparison pages is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--12","ques":"Find the comment thread on the phys.org article 'JWST spots two early black holes growing far faster than their galaxies' where user bob_c replied to a top-level comment by alice_j. Report the full text of bob_c's reply.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_12.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open the named article and inspect its comment thread.\n- The final answer must reproduce the full reply by bob_c beneath alice_j's top-level comment.\n- An empty, partial, or unsupported answer is a FAIL."} -{"web_name":"Phys.org","id":"Phys.org--13","ques":"Register a new account on phys.org with username 'qa_explorer', email 'qa_explorer@example.com', full name 'QA Explorer', password 'BenchmarkPass2026'. Then open Account Settings, set the Location field to 'Berlin, Germany', and save. Report the username that appears in the page header after saving.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_13.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit registration and Account Settings.\n- The after-state must contain the newly registered account with every requested identity field and saved location.\n- The final answer must report the username shown after saving; self-report without the exact database state is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--13","ques":"Register a new account on phys.org with username 'qa_explorer', email 'qa_explorer@example.com', full name 'QA Explorer', and a non-empty password of your choice. Then open Account Settings, set the Location field to 'Berlin, Germany', and save. Report the username that appears in the page header after saving.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_13.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit registration and Account Settings.\n- The after-state must contain the newly registered account with every requested profile field, saved location, and a valid password hash.\n- The final answer must report the username shown after saving; self-report without the exact database state is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--14","ques":"Sign in as alice.j@test.com (password TestPass123!), open the article 'How a single star can reshape an entire galaxy', and remove it from her saved articles. Then visit the saved-articles page and report (a) how many items remain in the saved list and (b) the title of the most-recently-saved article shown at the top of the list.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_14.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, open the named article, remove it, and visit the saved list.\n- The after-state must equal the initial saved list with only that article removed.\n- The final answer must report both the remaining count and the top saved title; self-report without the database change is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--15","ques":"On the phys.org homepage, open the third article in the 'Trending now' sidebar. From its detail page, follow its category and switch that category to 'Popular'. Report the article title, its category, and its rank in that category's Popular list.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_15.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must start from the homepage, open the third Trending now article, and then open its category with the Popular sort.\n- The final answer must report the selected article title, category, and Popular-list rank.\n- An empty answer or an answer without the cross-page ranking check is a FAIL."} -{"web_name":"Phys.org","id":"Phys.org--16","ques":"On phys.org, search for 'CO2' and use the category filter to restrict to Chemistry. Among the filtered results, identify the article that mentions 'polyionic liquids' and report its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_16.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must search for CO2 with the Chemistry filter and open the matching polyionic-liquids article.\n- The final answer must report the source journal on that article.\n- An empty answer or an answer without the filtered search and detail page is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--16","ques":"On phys.org, search for 'CO2 systems' and use the category filter to restrict to Chemistry. Among the filtered results, identify the article that mentions 'polyionic liquids' and report its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_16.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must search for CO2 systems with the Chemistry filter and open the matching polyionic-liquids article.\n- The final answer must report the source journal on that article.\n- An empty answer or an answer without the filtered search and detail page is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--17","ques":"Sign in as alice.j@test.com (password TestPass123!), open Account Settings, and report the most recent search query shown in the user's recent search history widget.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_17.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in and visit Account Settings without changing the user's search history first.\n- The final answer must report the first query shown in the recent-search-history widget.\n- An empty answer or an answer without Account Settings evidence is a FAIL."} diff --git a/sites/phys_org/verify/test_verifiers.py b/sites/phys_org/verify/test_verifiers.py index 18356ffa..d8a7755f 100644 --- a/sites/phys_org/verify/test_verifiers.py +++ b/sites/phys_org/verify/test_verifiers.py @@ -31,6 +31,7 @@ class Case: urls: tuple[str, ...] answer: str stateful: bool = False + login_email: str | None = None MAGNETIC = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" @@ -52,6 +53,7 @@ class Case: ) JWST = "jwst-spots-two-early-black-holes-growing-far-faster-than-their-galaxie" CO2_ARTICLE = "anion-swap-unlocks-sevenfold-co-capture-in-polyionic-liquids" +VALID_BCRYPT_HASH = "$2b$12$zV7HfiJmZTqLsgP30kyvJemamXfJyBv66FPuQOrwYXXsyQvrafvie" CASES = { @@ -59,15 +61,17 @@ class Case: 1: Case((f"/article/{QUANTUM_CIRCUIT}",), "Technion"), 2: Case(("/search?q=quantum", f"/article/{TINY_ENERGY}"), "Nature Photonics"), 3: Case(("/trending", f"/article/{TOP_TRENDING}"), "Elena Yamamoto"), - 4: Case(("/login", "/saved"), "4 Astronomy & Space saved articles"), - 5: Case(("/login", "/saved", f"/article/{PITCH_ARTICLE}"), "Advanced Engineering Materials"), + 4: Case(("/login", "/saved"), "4 Astronomy & Space saved articles", + login_email="alice.j@test.com"), + 5: Case(("/login", "/saved", f"/article/{PITCH_ARTICLE}"), "Advanced Engineering Materials", + login_email="bob.c@test.com"), 6: Case(("/login", "/category/biology", f"/article/{BIOLOGY_ARTICLE}"), "Swapping molecular building blocks one by one reveals how receptors tell adrenaline from dopamine", stateful=True), 7: Case(("/login", "/category/nanotechnology", f"/article/{NANO_ARTICLE}", "/saved"), "RNA-built droplets create customizable organelles inside living cells", stateful=True), 8: Case(("/user/carol_d",), "3 comments"), - 9: Case(("/search?q=graphene", f"/article/{GRAPHENE_RECENT}", f"/article/{GRAPHENE_EARLIER}"), + 9: Case(("/search?q=graphene+systems", f"/article/{GRAPHENE_RECENT}", f"/article/{GRAPHENE_EARLIER}"), "Hourglass nanographenes unlock strong, robust multi-spin entanglement was earlier — Nano Letters"), 10: Case(("/category/astronomy?sort=popular", f"/article/{STAR_ARTICLE}"), "How a single star can reshape an entire galaxy"), @@ -79,29 +83,78 @@ class Case: 14: Case(("/login", f"/article/{STAR_ARTICLE}", "/saved"), "5 remain; More Star Wars-like worlds emerge as 27 planet candidates with two suns discovered is first", stateful=True), - 15: Case(("/", f"/article/{MAGNETIC}", "/category/physics?sort=popular"), + 15: Case(("/", f"/article/{MAGNETIC}", "/category/physics", + "/category/physics?sort=popular"), "Magnetic checkerboard separates microparticles by size and sends them along different paths; Physics; rank 1"), - 16: Case(("/search?q=CO2&category=chemistry", f"/article/{CO2_ARTICLE}"), + 16: Case(("/search?q=CO2+systems&category=chemistry", f"/article/{CO2_ARTICLE}"), "Journal of the American Chemical Society"), - 17: Case(("/login", "/account"), "exoplanet atmosphere"), + 17: Case(("/login", "/account"), "exoplanet atmosphere", + login_email="alice.j@test.com"), } -def _trajectory(run_dir: Path, task_id: int, urls: tuple[str, ...], answer: str) -> None: +def _trajectory(run_dir: Path, task_id: int, urls: tuple[str, ...], answer: str, + login_email: str | None = None, + login_email_overwrite: str | None = None, + login_search_text: str | None = None, + actions: tuple[str, ...] | None = None, + base_url: str = BASE_URL) -> None: screenshots = run_dir / "screenshots" screenshots.mkdir(parents=True) steps = [] for index, suffix in enumerate(urls): - url = BASE_URL + suffix + url = base_url + suffix steps.append({ "step": index, "url": url, "title": "Phys.org Mirror", - "action": "done" if index == len(urls) - 1 else "click", + "action": (actions[index] if actions is not None else + ("done" if index == len(urls) - 1 else "click")), "params": {}, "screenshot_before": f"step_{index:03d}.png", "screenshot_after": f"step_{index + 1:03d}.png", }) + if login_email: + steps.insert(1, { + "step": 1, + "url": base_url + "/login", + "title": "Phys.org Mirror", + "action": "input", + "params": {"index": 1, "text": login_email}, + "screenshot_before": "login_email_before.png", + "screenshot_after": "login_email_after.png", + }) + steps.insert(2, { + "step": 2, + "url": base_url + "/login", + "title": "Phys.org Mirror", + "action": "input", + "params": {"index": 2, "text": "TestPass123!"}, + "screenshot_before": "login_password_before.png", + "screenshot_after": "login_password_after.png", + }) + if login_search_text is not None: + steps.insert(1, { + "step": 1, + "url": base_url + "/login", + "title": "Phys.org Mirror", + "action": "input", + "params": {"index": 0, "text": login_search_text}, + "screenshot_before": "header_search_before.png", + "screenshot_after": "header_search_after.png", + }) + if login_email_overwrite is not None: + steps.insert(2, { + "step": 2, + "url": base_url + "/login", + "title": "Phys.org Mirror", + "action": "input", + "params": {"index": 1, "text": login_email_overwrite}, + "screenshot_before": "login_email_overwrite_before.png", + "screenshot_after": "login_email_overwrite_after.png", + }) + for index, step in enumerate(steps): + step["step"] = index payload = { "task": f"contract fixture for Phys.org--{task_id}", "task_id": f"Phys.org--{task_id}", @@ -116,7 +169,9 @@ def _trajectory(run_dir: Path, task_id: int, urls: tuple[str, ...], answer: str) (run_dir / "trajectory.json").write_text(json.dumps(payload), encoding="utf-8") -def _mutate_after_db(task_id: int, db_path: Path) -> None: +def _mutate_after_db(task_id: int, db_path: Path, + registration_password_hash: str = VALID_BCRYPT_HASH, + task_7_resave: bool = False) -> None: con = sqlite3.connect(db_path) try: if task_id == 6: @@ -127,17 +182,26 @@ def _mutate_after_db(task_id: int, db_path: Path) -> None: ("Reviewed for our weekly journal club", BIOLOGY_ARTICLE), ) elif task_id == 7: - con.execute( - "INSERT INTO saved_articles(user_id,article_id,note,created_at) " - "SELECT u.id,a.id,?,'2026-08-15 12:00:00' " - "FROM users u, articles a WHERE u.username='david_k' AND a.slug=?", - ("Compare with our process", NANO_ARTICLE), - ) + if task_7_resave: + con.execute( + "UPDATE saved_articles SET note=?,created_at='2026-08-15 12:00:00' " + "WHERE user_id=(SELECT id FROM users WHERE username='david_k') " + "AND article_id=(SELECT id FROM articles WHERE slug=?)", + ("Compare with our process", NANO_ARTICLE), + ) + else: + con.execute( + "INSERT INTO saved_articles(user_id,article_id,note,created_at) " + "SELECT u.id,a.id,?,'2026-08-15 12:00:00' " + "FROM users u, articles a WHERE u.username='david_k' AND a.slug=?", + ("Compare with our process", NANO_ARTICLE), + ) elif task_id == 13: con.execute( "INSERT INTO users(username,email,password_hash,full_name,bio,location,interests,created_at) " - "VALUES('qa_explorer','qa_explorer@example.com','test-hash','QA Explorer',''," - "'Berlin, Germany','','2026-08-15 12:00:00')" + "VALUES('qa_explorer','qa_explorer@example.com',?,'QA Explorer',''," + "'Berlin, Germany','','2026-08-15 12:00:00')", + (registration_password_hash,), ) elif task_id == 14: con.execute( @@ -168,14 +232,22 @@ def tearDown(self) -> None: shutil.rmtree(self.temp_dir) def _run(self, task_id: int, urls: tuple[str, ...], answer: str, - *, mutate_state: bool = False) -> subprocess.CompletedProcess[str]: + *, mutate_state: bool = False, + login_email: str | None = None, + login_email_overwrite: str | None = None, + login_search_text: str | None = None, + registration_password_hash: str = VALID_BCRYPT_HASH, + task_7_resave: bool = False, + actions: tuple[str, ...] | None = None, + base_url: str = BASE_URL) -> subprocess.CompletedProcess[str]: run_dir = self.temp_dir / f"run-{task_id}-{len(list(self.temp_dir.glob('run-*')))}" run_dir.mkdir() - _trajectory(run_dir, task_id, urls, answer) + _trajectory(run_dir, task_id, urls, answer, login_email, + login_email_overwrite, login_search_text, actions, base_url) after_db = run_dir / "after.db" - shutil.copy2(SEED_DB, after_db) + shutil.copy2(self.initial_db, after_db) if mutate_state: - _mutate_after_db(task_id, after_db) + _mutate_after_db(task_id, after_db, registration_password_hash, task_7_resave) verifier = VERIFY_DIR / f"verify_{task_id}.py" return subprocess.run( [sys.executable, str(verifier), "--run_dir", str(run_dir), @@ -206,6 +278,10 @@ def test_task_metadata_declares_all_grading_artifacts(self) -> None: self.assertNotIn("answer", row) self.assertNotIn("count shown next to the search term", rows[9]["ques"]) self.assertIn("Popular", rows[15]["ques"]) + self.assertIn("password of your choice", rows[13]["ques"]) + self.assertNotIn("BenchmarkPass2026", rows[13]["ques"]) + self.assertIn("graphene systems", rows[9]["ques"]) + self.assertIn("CO2 systems", rows[16]["ques"]) def test_all_verifiers_reject_no_op(self) -> None: for task_id in range(18): @@ -217,9 +293,63 @@ def test_all_verifiers_accept_correct_run(self) -> None: for task_id, case in CASES.items(): with self.subTest(task=task_id): result = self._run(task_id, case.urls, case.answer, - mutate_state=case.stateful) + mutate_state=case.stateful, + login_email=case.login_email) self.assert_verdict(result, True) + def test_login_tasks_reject_wrong_account(self) -> None: + for task_id in (4, 5, 17): + case = CASES[task_id] + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, case.answer, + login_email="wrong.user@example.com") + self.assert_verdict(result, False) + + def test_login_tasks_reject_overwritten_expected_email(self) -> None: + for task_id in (4, 5, 17): + case = CASES[task_id] + with self.subTest(task=task_id): + result = self._run( + task_id, case.urls, case.answer, + login_email=case.login_email, + login_email_overwrite="wrong.user@example.com", + ) + self.assert_verdict(result, False) + + def test_login_tasks_reject_expected_email_typed_into_search(self) -> None: + for task_id in (4, 5, 17): + case = CASES[task_id] + with self.subTest(task=task_id): + result = self._run( + task_id, case.urls, case.answer, + login_email="wrong.user@example.com", + login_search_text=case.login_email, + ) + self.assert_verdict(result, False) + + def test_registration_rejects_missing_password_hash(self) -> None: + case = CASES[13] + result = self._run(13, case.urls, case.answer, mutate_state=True, + registration_password_hash="") + self.assert_verdict(result, False) + + def test_save_task_rejects_resaving_an_initially_saved_article(self) -> None: + con = sqlite3.connect(self.initial_db) + try: + con.execute( + "INSERT INTO saved_articles(user_id,article_id,note,created_at) " + "SELECT u.id,a.id,'Existing note','2026-08-14 12:00:00' " + "FROM users u, articles a WHERE u.username='david_k' AND a.slug=?", + (NANO_ARTICLE,), + ) + con.commit() + finally: + con.close() + case = CASES[7] + result = self._run(7, case.urls, case.answer, mutate_state=True, + task_7_resave=True) + self.assert_verdict(result, False) + def test_task_11_accepts_direct_winner_title(self) -> None: result = self._run(11, CASES[11].urls, QUANTUM_GEOMETRY_TITLE) self.assert_verdict(result, True) @@ -245,7 +375,72 @@ def test_all_verifiers_reject_wrong_answer(self) -> None: for task_id, case in CASES.items(): with self.subTest(task=task_id): result = self._run(task_id, case.urls, "incorrect answer", - mutate_state=case.stateful) + mutate_state=case.stateful, + login_email=case.login_email) + self.assert_verdict(result, False) + + def test_all_verifiers_reject_negated_correct_answer(self) -> None: + for task_id, case in CASES.items(): + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, f"Not {case.answer}", + mutate_state=case.stateful, + login_email=case.login_email) + self.assert_verdict(result, False) + + def test_negation_scope_stops_at_sentence_boundary(self) -> None: + case = CASES[0] + answer = "Not visible. Reviews of Modern Physics" + result = self._run(0, case.urls, answer) + self.assert_verdict(result, True) + + def test_answer_corrections_use_the_final_claim(self) -> None: + case = CASES[0] + accepted = [ + "Not Reviews of Modern Physics? Correction: Reviews of Modern Physics.", + "No, Reviews of Modern Physics is the source journal.", + "I did not stop at search and found Reviews of Modern Physics.", + ] + rejected = [ + "It is not in any reasonable sense actually Reviews of Modern Physics.", + "Reviews of Modern Physics, but not Reviews of Modern Physics.", + "Reviews of Modern Physics is not the source journal.", + "Reviews of Modern Physics is definitely not the source journal.", + "Reviews of Modern Physics? No, that is wrong.", + "Reviews of Modern Physics — definitely not the source.", + "Reviews of Modern Physics, however, is not the source.", + "Reviews of Modern Physics; no, that is wrong.", + ] + for answer in accepted: + with self.subTest(answer=answer): + self.assert_verdict(self._run(0, case.urls, answer), True) + for answer in rejected: + with self.subTest(answer=answer): + self.assert_verdict(self._run(0, case.urls, answer), False) + + def test_task_15_rejects_direct_navigation_between_checkpoints(self) -> None: + case = CASES[15] + result = self._run( + 15, case.urls, case.answer, + actions=("navigate", "navigate", "navigate", "done"), + ) + self.assert_verdict(result, False) + + def test_all_verifiers_reject_wrong_host_navigation(self) -> None: + for task_id, case in CASES.items(): + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, case.answer, + mutate_state=case.stateful, + login_email=case.login_email, + base_url="https://example.com") + self.assert_verdict(result, False) + + def test_all_verifiers_reject_navigation_outside_start_origin(self) -> None: + for task_id, case in CASES.items(): + with self.subTest(task=task_id): + result = self._run(task_id, case.urls, case.answer, + mutate_state=case.stateful, + login_email=case.login_email, + base_url="http://localhost:40015") self.assert_verdict(result, False) def test_stateful_verifiers_reject_unchanged_db(self) -> None: @@ -268,6 +463,77 @@ def test_comparison_verifiers_reject_reversed_claim(self) -> None: result = self._run(task_id, CASES[task_id].urls, answer) self.assert_verdict(result, False) + def test_comparison_relation_is_bound_to_the_expected_title(self) -> None: + task_11_answers = { + False: ( + f"{QUANTUM_GEOMETRY_TITLE} was later; " + f"{MAGNETIC_TITLE} was earlier." + ), + True: ( + f"{QUANTUM_GEOMETRY_TITLE} was earlier; " + f"{MAGNETIC_TITLE} was later." + ), + } + for expected, answer in task_11_answers.items(): + with self.subTest(task=11, expected=expected): + self.assert_verdict( + self._run(11, CASES[11].urls, answer), expected + ) + task_9_wrong = ( + "Hourglass nanographenes unlock strong, robust multi-spin entanglement " + "was later and Machine learning proves that graphene is hydrophobic " + "was earlier; Nano Letters." + ) + self.assert_verdict(self._run(9, CASES[9].urls, task_9_wrong), False) + + def test_comparison_understands_pair_direction_and_pronouns(self) -> None: + task_11_answers = { + False: ( + f"{MAGNETIC_TITLE} was published earlier than " + f"{QUANTUM_GEOMETRY_TITLE}." + ), + True: ( + f"{MAGNETIC_TITLE} was published later than " + f"{QUANTUM_GEOMETRY_TITLE}." + ), + } + for expected, answer in task_11_answers.items(): + with self.subTest(kind="than", expected=expected): + self.assert_verdict( + self._run(11, CASES[11].urls, answer), expected + ) + accepted = [ + f"{QUANTUM_GEOMETRY_TITLE}, not {MAGNETIC_TITLE}, was published earlier.", + f"Between {QUANTUM_GEOMETRY_TITLE} and {MAGNETIC_TITLE}, " + "the former was published earlier.", + ] + for answer in accepted: + with self.subTest(kind="reference", answer=answer): + self.assert_verdict(self._run(11, CASES[11].urls, answer), True) + + def test_number_answer_rejects_direct_post_value_denial(self) -> None: + answer = "4 is definitely not the count." + self.assert_verdict(self._run(4, CASES[4].urls, answer, + login_email=CASES[4].login_email), False) + + def test_comparison_correction_uses_final_relation(self) -> None: + answer = ( + "It was not later. Correction: " + f"{QUANTUM_GEOMETRY_TITLE} was published earlier." + ) + result = self._run(11, CASES[11].urls, answer) + self.assert_verdict(result, True) + + def test_multihop_task_rejects_out_of_order_navigation(self) -> None: + case = CASES[15] + out_of_order = ( + "/category/physics?sort=popular", + f"/article/{MAGNETIC}", + "/", + ) + result = self._run(15, out_of_order, case.answer) + self.assert_verdict(result, False) + def test_recent_search_verifier_rejects_history_mutation(self) -> None: case = CASES[17] result = self._run(17, case.urls, case.answer, mutate_state=True) diff --git a/sites/phys_org/verify/verify_13.py b/sites/phys_org/verify/verify_13.py index 60aeae12..4681ae86 100644 --- a/sites/phys_org/verify/verify_13.py +++ b/sites/phys_org/verify/verify_13.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +import re + from verify_lib import (Judge, contains_all, db_query, final_answer, load_run, parse_args, resolve_db, visited_path) QUERY = """ -SELECT username,email,full_name,location +SELECT username,email,full_name,location,password_hash FROM users WHERE username='qa_explorer' OR email='qa_explorer@example.com' """ EXPECTED = ("qa_explorer", "qa_explorer@example.com", "QA Explorer", "Berlin, Germany") +BCRYPT_HASH = re.compile(r"\$2[aby]\$\d{2}\$[./A-Za-z0-9]{53}") def main(): args = parse_args() @@ -15,11 +18,20 @@ def main(): answer = final_answer(trajectory) initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY) after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY) + after_profiles = [] if after is None else [row[:4] for row in after] + valid_password_hash = ( + after is not None + and len(after) == 1 + and BCRYPT_HASH.fullmatch(str(after[0][4] or "")) is not None + ) judge = Judge("Phys.org--13") judge.check("nav_register", visited_path(trajectory, "/register"), "visited registration") judge.check("nav_account", visited_path(trajectory, "/account"), "visited Account Settings") judge.check("db_user_absent_initially", initial == [], f"initial_rows={initial}") - judge.check("db_registered_profile_exact", after == [EXPECTED], f"after_rows={after}") + judge.check("db_registered_profile_exact", after_profiles == [EXPECTED], + f"after_profiles={after_profiles}") + judge.check("db_password_hash_present", valid_password_hash, + "registered account has a valid bcrypt password hash") judge.check("answer_username", contains_all(answer, ["qa_explorer"]), repr(answer)) judge.emit() diff --git a/sites/phys_org/verify/verify_15.py b/sites/phys_org/verify/verify_15.py index 0b551af6..be838b97 100644 --- a/sites/phys_org/verify/verify_15.py +++ b/sites/phys_org/verify/verify_15.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -from verify_lib import contains_all, has_number, run_stateless, visited_category, visited_path +from verify_lib import (clicked_path_transition, contains_all, has_number, + run_stateless, visited_category, visited_path) SLUG = "magnetic-checkerboard-separates-microparticles-by-size-and-sends-them-" TITLE = "Magnetic checkerboard separates microparticles by size and sends them along different paths" @@ -9,6 +10,16 @@ def checks(t, answer): ("nav_home", visited_path(t, "/"), "visited homepage sidebar"), ("nav_third_trending_article", visited_path(t, f"/article/{SLUG}"), "opened third sidebar entry"), ("nav_physics_popular", visited_category(t, "physics", "popular"), "opened category Popular view"), + ("click_third_trending", + clicked_path_transition(t, "/", f"/article/{SLUG}"), + "clicked from home into the third Trending article"), + ("click_article_category", + clicked_path_transition(t, f"/article/{SLUG}", "/category/physics"), + "followed the Physics category link from the article"), + ("click_category_popular", + clicked_path_transition(t, "/category/physics", "/category/physics", + {"sort": "popular"}), + "switched the category view to Popular by click"), ], [ ("answer_title_category_rank", contains_all(answer, [TITLE, "Physics"]) and has_number(answer, 1), repr(answer)), diff --git a/sites/phys_org/verify/verify_16.py b/sites/phys_org/verify/verify_16.py index 8ac8fd89..f3410448 100644 --- a/sites/phys_org/verify/verify_16.py +++ b/sites/phys_org/verify/verify_16.py @@ -5,8 +5,8 @@ def checks(t, answer): return ([ - ("nav_filtered_co2_search", visited_search(t, "CO2", "chemistry"), - "searched CO2 with Chemistry filter"), + ("nav_filtered_co2_systems_search", visited_search(t, "CO2 systems", "chemistry"), + "searched CO2 systems with Chemistry filter"), ("nav_target_article", visited_path(t, f"/article/{SLUG}"), "opened target article"), ], [("answer_source_journal", contains_all(answer, ["Journal of the American Chemical Society"]), repr(answer))]) diff --git a/sites/phys_org/verify/verify_17.py b/sites/phys_org/verify/verify_17.py index 2bcf2c54..81c5f8fb 100644 --- a/sites/phys_org/verify/verify_17.py +++ b/sites/phys_org/verify/verify_17.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from verify_lib import (Judge, contains_all, db_query, final_answer, load_run, - parse_args, resolve_db, visited_path) +from verify_lib import (Judge, contains_all, db_query, filled_field, final_answer, + load_run, parse_args, resolve_db, visited_path) QUERY = """ SELECT s.query,s.created_at @@ -18,6 +18,9 @@ def main(): after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY) judge = Judge("Phys.org--17") judge.check("nav_login", visited_path(trajectory, "/login"), "visited login") + judge.check("login_as_alice", + filled_field(trajectory, "email", "alice.j@test.com", "/login"), + "filled Alice's email on login") judge.check("nav_account", visited_path(trajectory, "/account"), "visited Account Settings") judge.check("db_search_history_unchanged", initial is not None and after == initial, f"initial_history={initial} after_history={after}") diff --git a/sites/phys_org/verify/verify_4.py b/sites/phys_org/verify/verify_4.py index 523ff029..2dc387cc 100644 --- a/sites/phys_org/verify/verify_4.py +++ b/sites/phys_org/verify/verify_4.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -from verify_lib import has_number, run_stateless, visited_path +from verify_lib import filled_field, has_number, run_stateless, visited_path def checks(t, answer): return ([ ("nav_login", visited_path(t, "/login"), "visited login"), + ("login_as_alice", filled_field(t, "email", "alice.j@test.com", "/login"), + "filled Alice's email on login"), ("nav_saved", visited_path(t, "/saved"), "visited saved list"), ], [("answer_count_four", has_number(answer, 4), repr(answer))]) diff --git a/sites/phys_org/verify/verify_5.py b/sites/phys_org/verify/verify_5.py index a6f97005..0803e13c 100644 --- a/sites/phys_org/verify/verify_5.py +++ b/sites/phys_org/verify/verify_5.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -from verify_lib import contains_all, run_stateless, visited_path +from verify_lib import contains_all, filled_field, run_stateless, visited_path SLUG = "cracking-the-code-of-hypersonic-flight-a-decade-of-experiments-maps-tu" def checks(t, answer): return ([ ("nav_login", visited_path(t, "/login"), "visited login"), + ("login_as_bob", filled_field(t, "email", "bob.c@test.com", "/login"), + "filled Bob's email on login"), ("nav_saved", visited_path(t, "/saved"), "visited saved list"), ("nav_noted_article", visited_path(t, f"/article/{SLUG}"), "opened noted article"), ], [("answer_source_journal", contains_all(answer, ["Advanced Engineering Materials"]), repr(answer))]) diff --git a/sites/phys_org/verify/verify_7.py b/sites/phys_org/verify/verify_7.py index 5c591be3..acb1815f 100644 --- a/sites/phys_org/verify/verify_7.py +++ b/sites/phys_org/verify/verify_7.py @@ -3,7 +3,14 @@ parse_args, resolve_db, visited_category, visited_path) NOTE = "Compare with our process" -QUERY = """ +INITIAL_QUERY = """ +SELECT a.id +FROM saved_articles s +JOIN users u ON u.id=s.user_id +JOIN articles a ON a.id=s.article_id +WHERE u.username='david_k' +""" +AFTER_QUERY = """ SELECT a.id, a.title, a.slug FROM saved_articles s JOIN users u ON u.id=s.user_id @@ -16,8 +23,8 @@ def main(): args = parse_args() trajectory = load_run(args.run_dir) answer = final_answer(trajectory) - initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), QUERY, (NOTE,)) - after = db_query(resolve_db(args.after_db, args.container, "instance"), QUERY, (NOTE,)) + initial = db_query(resolve_db(args.initial_db, args.container, "instance_seed"), INITIAL_QUERY) + after = db_query(resolve_db(args.after_db, args.container, "instance"), AFTER_QUERY, (NOTE,)) initial_ids = set() if initial is None else {row[0] for row in initial} new_rows = [] if after is None else [row for row in after if row[0] not in initial_ids] judge = Judge("Phys.org--7") diff --git a/sites/phys_org/verify/verify_9.py b/sites/phys_org/verify/verify_9.py index 7b071ccb..31b40a9a 100644 --- a/sites/phys_org/verify/verify_9.py +++ b/sites/phys_org/verify/verify_9.py @@ -3,15 +3,18 @@ RECENT = "machine-learning-proves-that-graphene-is-hydrophobic" EARLIER = "hourglass-nanographenes-unlock-strong-robust-multi-spin-entanglement" +RECENT_TITLE = "Machine learning proves that graphene is hydrophobic" +EARLIER_TITLE = "Hourglass nanographenes unlock strong, robust multi-spin entanglement" def checks(t, answer): return ([ - ("nav_graphene_search", visited_search(t, "graphene"), "searched for graphene"), + ("nav_graphene_systems_search", visited_search(t, "graphene systems"), + "searched for graphene systems"), ("nav_recent_nanotech_result", visited_path(t, f"/article/{RECENT}"), "opened first comparison article"), ("nav_earlier_nanotech_result", visited_path(t, f"/article/{EARLIER}"), "opened second comparison article"), ], [ ("answer_earlier_article_and_journal", - claims_earlier(answer, "Hourglass nanographenes unlock strong, robust multi-spin entanglement") + claims_earlier(answer, EARLIER_TITLE, RECENT_TITLE) and contains_all(answer, ["Nano Letters"]), repr(answer)), ]) diff --git a/sites/phys_org/verify/verify_lib.py b/sites/phys_org/verify/verify_lib.py index 4be45ebf..3b683096 100755 --- a/sites/phys_org/verify/verify_lib.py +++ b/sites/phys_org/verify/verify_lib.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import ipaddress import json import os import re @@ -30,12 +31,38 @@ def step_urls(trajectory: dict) -> list[str]: return [str(step.get("url", "")) for step in trajectory.get("steps", [])] +def _is_loopback_host(hostname: str) -> bool: + if hostname.casefold() == "localhost": + return True + try: + return ipaddress.ip_address(hostname).is_loopback + except ValueError: + return False + + +def _is_mirror_url(url: str, trajectory: dict) -> bool: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + return False + start = urlparse(str(trajectory.get("start_url") or "")) + if not start.hostname or not _is_loopback_host(start.hostname): + return False + return ( + _is_loopback_host(parsed.hostname) + and parsed.scheme == start.scheme + and parsed.port == start.port + ) + + def visited_path(trajectory: dict, path: str) -> bool: - return any(urlparse(url).path == path for url in step_urls(trajectory)) + return any(_is_mirror_url(url, trajectory) and urlparse(url).path == path + for url in step_urls(trajectory)) def visited_search(trajectory: dict, query: str, category: str | None = None) -> bool: for url in step_urls(trajectory): + if not _is_mirror_url(url, trajectory): + continue parsed = urlparse(url) if parsed.path != "/search": continue @@ -51,6 +78,8 @@ def visited_search(trajectory: dict, query: str, category: str | None = None) -> def visited_category(trajectory: dict, slug: str, sort: str | None = None) -> bool: path = f"/category/{slug}" for url in step_urls(trajectory): + if not _is_mirror_url(url, trajectory): + continue parsed = urlparse(url) if parsed.path != path: continue @@ -62,37 +91,187 @@ def visited_category(trajectory: dict, slug: str, sort: str | None = None) -> bo return False +def visited_in_order(trajectory: dict, + requirements: list[tuple[str, dict[str, str]]]) -> bool: + """Require URL path/query checkpoints to appear in trajectory order.""" + urls = step_urls(trajectory) + cursor = 0 + for path, expected_query in requirements: + matched = False + for index in range(cursor, len(urls)): + if not _is_mirror_url(urls[index], trajectory): + continue + parsed = urlparse(urls[index]) + params = parse_qs(parsed.query) + query_matches = all( + norm((params.get(key) or [""])[0]) == norm(value) + for key, value in expected_query.items() + ) + if parsed.path == path and query_matches: + cursor = index + 1 + matched = True + break + if not matched: + return False + return True + + +def clicked_path_transition(trajectory: dict, from_path: str, to_path: str, + to_query: dict[str, str] | None = None) -> bool: + """Require an adjacent same-origin transition caused by a click action.""" + steps = trajectory.get("steps", []) + expected_query = to_query or {} + for current, following in zip(steps, steps[1:]): + current_url = str(current.get("url", "")) + following_url = str(following.get("url", "")) + if not (_is_mirror_url(current_url, trajectory) + and _is_mirror_url(following_url, trajectory)): + continue + if urlparse(current_url).path != from_path: + continue + if norm(current.get("action")) != "click": + continue + parsed_following = urlparse(following_url) + if parsed_following.path != to_path: + continue + params = parse_qs(parsed_following.query) + if all(norm((params.get(key) or [""])[0]) == norm(value) + for key, value in expected_query.items()): + return True + return False + + def final_answer(trajectory: dict) -> str: return str(trajectory.get("final_answer") or "").strip() +def filled_field(trajectory: dict, field: str, expected: str, + path: str | None = None) -> bool: + """Return whether a named field's final recorded value matches exactly. + + Legacy probe trajectories identify fields by CSS selector. The repository + runner records only ``input(index, text)``. The fixed login page has a + global search input before the form, then email and password; among the + form inputs used to authenticate, email is therefore the penultimate DOM + index. In both schemas the last value for that field wins, so an + overwritten credential cannot pass. + """ + field_pattern = re.compile(rf"(?:name\s*=\s*['\"]?{re.escape(field)}\b|#{re.escape(field)}\b)", + re.IGNORECASE) + legacy_values: list[str] = [] + indexed_values: dict[int, list[str]] = {} + for step in trajectory.get("steps", []): + action = norm(step.get("action")) + if action not in {"fill", "type", "input"}: + continue + step_url = str(step.get("url", "")) + if not _is_mirror_url(step_url, trajectory): + continue + if path is not None and urlparse(step_url).path != path: + continue + params = step.get("params") or {} + value = params.get("text", params.get("value", "")) + if action in {"fill", "type"}: + selector = str(params.get("css") or params.get("selector") or "") + if field_pattern.search(selector): + legacy_values.append(norm(value)) + continue + try: + index = int(params.get("index")) + except (TypeError, ValueError): + continue + indexed_values.setdefault(index, []).append(norm(value)) + if legacy_values: + return legacy_values[-1] == norm(expected) + if field == "email" and len(indexed_values) >= 2: + email_index = sorted(indexed_values)[-2] + return indexed_values[email_index][-1] == norm(expected) + return False + + def norm(value: object) -> str: text = unicodedata.normalize("NFKC", str(value or "")) return re.sub(r"\s+", " ", text).strip().casefold() -def contains_all(text: str, expected: list[str] | tuple[str, ...]) -> bool: +NEGATION_WORDS = { + "not", "no", "never", "without", "isn't", "isnt", "aren't", "arent", + "wasn't", "wasnt", "weren't", "werent", "doesn't", "doesnt", "didn't", + "didnt", +} + + +def _negated_at(normalized: str, start: int) -> bool: + prefix = normalized[:start] + clause = re.split( + r"(?:[.!?;:\n]+|\b(?:and|but|however|instead)\b)", prefix + )[-1] + if re.fullmatch(r"\s*no\s*,\s*", clause): + return False + prefix_words = re.findall(r"[a-z0-9]+(?:['’][a-z]+)?", clause) + return any(word in NEGATION_WORDS for word in prefix_words) + + +def _denied_after(normalized: str, end: int) -> bool: + """Detect a direct post-value denial such as ``X is not the answer``.""" + suffix = normalized[end:] + for _ in range(4): + stripped = re.sub( + r"^\s*(?:[-—–,:;!?]+|\bhowever\b)\s*", "", suffix + ) + if stripped == suffix: + break + suffix = stripped + return re.match( + r"\s*(?:no\b|(?:[a-z]+\s+){1,3}(?:not|never|no)\b|" + r"(?:isn't|isnt|aren't|arent|wasn't|wasnt|" + r"weren't|werent|doesn't|doesnt|don't|dont|didn't|didnt|can't|" + r"cant|couldn't|couldnt|wouldn't|wouldnt|shouldn't|shouldnt)\b)", + suffix, + ) is not None + + +def _contains_affirmatively(text: str, expected: object) -> bool: normalized = norm(text) - return all(norm(item) in normalized for item in expected) + needle = norm(expected) + if not needle: + return False + matches = list(re.finditer(re.escape(needle), normalized)) + if not matches: + return False + last = matches[-1] + return ( + not _negated_at(normalized, last.start()) + and not _denied_after(normalized, last.end()) + ) + + +def contains_all(text: str, expected: list[str] | tuple[str, ...]) -> bool: + return all(_contains_affirmatively(text, item) for item in expected) def contains_any(text: str, expected: list[str] | tuple[str, ...]) -> bool: - normalized = norm(text) - return any(norm(item) in normalized for item in expected) + return any(_contains_affirmatively(text, item) for item in expected) def has_number(text: str, value: int) -> bool: - return re.search(rf"(? bool: +def claims_earlier(text: str, expected_title: str, + other_title: str | None = None) -> bool: """Require an affirmative earlier/older claim and reject reversed wording.""" - normalized = norm(text) - if norm(expected_title) not in normalized: + if not _contains_affirmatively(text, expected_title): return False - if _reverses_earlier_claim(normalized): - return False - return re.search(r"\b(?:earlier|older|first|before)\b", normalized) is not None + return _relation_for_title(text, expected_title, other_title) is True def answers_earlier_comparison(text: str, expected_title: str, other_title: str) -> bool: @@ -102,21 +281,90 @@ def answers_earlier_comparison(text: str, expected_title: str, other_title: str) the winning title alone. If both titles are repeated, relational wording is still required so the answer cannot pass while remaining ambiguous. """ - if claims_earlier(text, expected_title): + relation = _relation_for_title(text, expected_title, other_title) + if _contains_affirmatively(text, expected_title) and relation is True: return True + if relation is False: + return False normalized = norm(text) return ( - norm(expected_title) in normalized + _contains_affirmatively(text, expected_title) and norm(other_title) not in normalized - and not _reverses_earlier_claim(normalized) ) -def _reverses_earlier_claim(normalized: str) -> bool: - return re.search( - r"\b(?:later|newer|after)\b|\bnot\s+(?:the\s+)?(?:earlier|older|first)\b", - normalized, - ) is not None +def _relation_for_title(text: str, expected_title: str, + other_title: str | None = None) -> bool | None: + """Return the relation claim local to a title, bounded by its comparator.""" + normalized = norm(text) + expected = norm(expected_title) + matches = list(re.finditer(re.escape(expected), normalized)) + if not matches: + return None + target = matches[-1] + other = norm(other_title) if other_title else "" + comparators = list(re.finditer(re.escape(other), normalized)) if other else [] + if comparators: + comparator = min( + comparators, + key=lambda match: min( + abs(match.end() - target.start()), + abs(match.start() - target.end()), + ), + ) + pair_first, pair_second = sorted( + (target, comparator), key=lambda match: match.start() + ) + expected_is_first = pair_first is target + between = normalized[pair_first.end():pair_second.start()] + between_claims = _earlier_relation_claims(between) + if between_claims and re.search(r"\bthan\s*$", between): + applies_to_expected = between_claims[-1] + return applies_to_expected if expected_is_first else not applies_to_expected + + sentence_end_match = re.search(r"[.!?;\n]", normalized[pair_second.end():]) + sentence_end = ( + pair_second.end() + sentence_end_match.start() + if sentence_end_match else len(normalized) + ) + after_pair = normalized[pair_second.end():sentence_end] + after_claims = _earlier_relation_claims(after_pair) + if re.search(r"\bnot\b", between) and after_claims: + applies_to_expected = after_claims[-1] + return applies_to_expected if expected_is_first else not applies_to_expected + reference = re.search(r"\b(former|latter)\b", after_pair) + if reference and after_claims: + refers_to_first = reference.group(1) == "former" + refers_to_expected = refers_to_first == expected_is_first + return after_claims[-1] if refers_to_expected else not after_claims[-1] + + left = 0 + right = len(normalized) + for boundary in re.finditer(r"[.!?;\n]+", normalized): + if boundary.end() <= target.start(): + left = max(left, boundary.end()) + elif boundary.start() >= target.end(): + right = min(right, boundary.start()) + break + if other: + for comparator in re.finditer(re.escape(other), normalized): + if comparator.end() <= target.start(): + left = max(left, comparator.end()) + elif comparator.start() >= target.end(): + right = min(right, comparator.start()) + break + claims = _earlier_relation_claims(normalized[left:right]) + return claims[-1] if claims else None + + +def _earlier_relation_claims(normalized: str) -> list[bool]: + """Return ordered relation claims; True means the answer asserts earlier.""" + claims: list[tuple[int, bool]] = [] + for match in re.finditer(r"\b(?:earlier|older|first|before)\b", normalized): + claims.append((match.start(), not _negated_at(normalized, match.start()))) + for match in re.finditer(r"\b(?:later|newer|after)\b", normalized): + claims.append((match.start(), _negated_at(normalized, match.start()))) + return [supports_earlier for _, supports_earlier in sorted(claims)] def fetch_db(container: str, kind: str) -> str: From 4c97fedbc57e550e5d8823e0314174446267f396 Mon Sep 17 00:00:00 2001 From: JackJin <1037461232@qq.com> Date: Tue, 18 Aug 2026 14:35:57 +0800 Subject: [PATCH 6/7] fix(phys_org): prevent mobile header overflow --- sites/phys_org/static/css/main.css | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/sites/phys_org/static/css/main.css b/sites/phys_org/static/css/main.css index 3aaf34c7..3ba020de 100644 --- a/sites/phys_org/static/css/main.css +++ b/sites/phys_org/static/css/main.css @@ -117,6 +117,34 @@ img { max-width: 100%; height: auto; display: block; } .nav-bar li a:hover { background: var(--c-accent); color: #fff; } .nav-bar li a.active { background: var(--c-accent); color: #fff; } +@media (max-width: 600px) { + .header-top { + flex-wrap: wrap; + gap: 8px 12px; + padding: 10px 16px; + } + .tagline { display: none; } + .header-search { + order: 3; + flex: 1 0 100%; + max-width: none; + min-width: 0; + margin-left: 0; + } + .header-search form, + .header-search input[type=text], + .header-search input[type=search] { + min-width: 0; + width: 100%; + } + .header-account { margin-left: auto; } + .nav-bar ul { padding: 0 8px; } + .nav-bar li a { + padding: 9px 10px; + font-size: 12px; + } +} + /* ---- Layout ---- */ .container { @@ -132,6 +160,22 @@ img { max-width: 100%; height: auto; display: block; } @media (max-width: 900px) { .layout { grid-template-columns: 1fr; } } +@media (max-width: 600px) { + .container { padding: 16px; } + .article-card { + grid-template-columns: 100px minmax(0, 1fr); + gap: 12px; + } + .article-card .thumb { + width: 100px; + height: 75px; + } + .article-detail h1 { font-size: 25px; } + .source-block dt { + display: block; + width: auto; + } +} /* ---- Cards & lists ---- */ From 810d814f80c0c6a84b9d3b7aea17d65c4e64c4b5 Mon Sep 17 00:00:00 2001 From: JackJin <1037461232@qq.com> Date: Tue, 18 Aug 2026 20:27:51 +0800 Subject: [PATCH 7/7] fix(phys_org): align task 10 grading contract --- AGENTS.md | 2 +- sites/phys_org/tasks.jsonl | 2 +- .../verify/test_environment_quality.py | 14 ++++++++++++++ sites/phys_org/verify/test_verifiers.py | 18 +++++++++++++++++- sites/phys_org/verify/verify_10.py | 4 ++-- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ad618907..9d6868df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,7 @@ docker run -d --rm --name wh-test \ curl -s http://localhost:8201/health | python3 -m json.tool | head # 5. every site renders 200 -for p in $(seq 41000 41014); do +for p in $(seq 41000 41016); do curl -so /dev/null -w "$p:%{http_code}\n" http://localhost:$p/ done diff --git a/sites/phys_org/tasks.jsonl b/sites/phys_org/tasks.jsonl index c3b240db..165ff0c5 100644 --- a/sites/phys_org/tasks.jsonl +++ b/sites/phys_org/tasks.jsonl @@ -8,7 +8,7 @@ {"web_name":"Phys.org","id":"Phys.org--7","ques":"Sign in as david.k@test.com (password TestPass123!), find a Nanotechnology article you have not yet saved, save it with the note 'Compare with our process', then verify it appears in your saved list and report the article title.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_7.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must sign in, visit Nanotechnology, open the chosen article, and return to the saved list.\n- The after-state must contain a newly saved Nanotechnology article for the requested user with the exact note.\n- The final answer must report the newly saved article title; self-report without the database mutation is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--8","ques":"Open the public profile page for user 'carol_d' on phys.org and report the count of their public comments shown in the profile stats.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_8.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit carol_d's public profile page.\n- The final answer must report the public comment count shown in the profile stats.\n- An empty answer or an answer without profile-page evidence is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--9","ques":"Search the phys.org mirror for 'graphene systems'. Among the results in the Nanotechnology category, compare the publication dates of 'Machine learning proves that graphene is hydrophobic' and 'Hourglass nanographenes unlock strong, robust multi-spin entanglement'. Report which was published earlier and its source journal.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_9.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must perform the graphene systems search and open both named Nanotechnology article detail pages.\n- The final answer must identify the earlier-published article and report its source journal.\n- An empty answer or an answer without both comparison pages is a FAIL."} -{"web_name":"Phys.org","id":"Phys.org--10","ques":"On the phys.org Astronomy & Space category page, sort by 'Popular' and report the title of the most-viewed article in that category.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_10.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open Astronomy & Space with the Popular sort and open the first-ranked article.\n- The final answer must report that article's full title.\n- An empty answer or an answer without the Popular view is a FAIL."} +{"web_name":"Phys.org","id":"Phys.org--10","ques":"On the phys.org Astronomy & Space category page, sort by 'Popular', open the most-viewed article, and report the institution listed as 'Provided by'.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_10.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open Astronomy & Space with the Popular sort and open the first-ranked article.\n- The final answer must report the institution shown in the article's Provided by field.\n- An empty answer or an answer without both navigation steps is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--11","ques":"Compare the publication dates of the two phys.org articles 'Magnetic checkerboard separates microparticles by size and sends them along different paths' and 'Quantum geometry applied to light-based systems expands toolkit for topological photonics'. Report which one was published earlier.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_11.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open both named article detail pages.\n- The final answer must identify which article has the earlier displayed publication date.\n- An empty answer or an answer without both comparison pages is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--12","ques":"Find the comment thread on the phys.org article 'JWST spots two early black holes growing far faster than their galaxies' where user bob_c replied to a top-level comment by alice_j. Report the full text of bob_c's reply.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_12.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must open the named article and inspect its comment thread.\n- The final answer must reproduce the full reply by bob_c beneath alice_j's top-level comment.\n- An empty, partial, or unsupported answer is a FAIL."} {"web_name":"Phys.org","id":"Phys.org--13","ques":"Register a new account on phys.org with username 'qa_explorer', email 'qa_explorer@example.com', full name 'QA Explorer', and a non-empty password of your choice. Then open Account Settings, set the Location field to 'Berlin, Germany', and save. Report the username that appears in the page header after saving.","web":"http://localhost:40016/","upstream_url":"https://phys.org/","verifier_path":"sites/phys_org/verify/verify_13.py","judge_rubric":"FACT CHECKPOINTS:\n- The trajectory must visit registration and Account Settings.\n- The after-state must contain the newly registered account with every requested profile field, saved location, and a valid password hash.\n- The final answer must report the username shown after saving; self-report without the exact database state is a FAIL."} diff --git a/sites/phys_org/verify/test_environment_quality.py b/sites/phys_org/verify/test_environment_quality.py index 188aae13..ca1abf8a 100644 --- a/sites/phys_org/verify/test_environment_quality.py +++ b/sites/phys_org/verify/test_environment_quality.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import re import sqlite3 import subprocess import tarfile @@ -25,6 +26,19 @@ def _load_seed_data(): class EnvironmentQualityTests(unittest.TestCase): + def test_agent_pre_pr_sweep_covers_every_registered_site(self) -> None: + agent_guide = (REPO_ROOT / "AGENTS.md").read_text(encoding="utf-8") + startup = (REPO_ROOT / "websyn_start.sh").read_text(encoding="utf-8") + site_match = re.search(r"SITES=\((.*?)\)", startup, re.DOTALL) + sweep_match = re.search( + r"for p in \$\(seq (\d+) (\d+)\); do", agent_guide + ) + self.assertIsNotNone(site_match) + self.assertIsNotNone(sweep_match) + sites = site_match.group(1).split() + sweep_start, sweep_end = map(int, sweep_match.groups()) + self.assertEqual((41000, 41000 + len(sites) - 1), (sweep_start, sweep_end)) + def test_subsection_journal_fix_preserves_legacy_institutions(self) -> None: seed_data = _load_seed_data() cases = [ diff --git a/sites/phys_org/verify/test_verifiers.py b/sites/phys_org/verify/test_verifiers.py index d8a7755f..cf59ecc8 100644 --- a/sites/phys_org/verify/test_verifiers.py +++ b/sites/phys_org/verify/test_verifiers.py @@ -74,7 +74,7 @@ class Case: 9: Case(("/search?q=graphene+systems", f"/article/{GRAPHENE_RECENT}", f"/article/{GRAPHENE_EARLIER}"), "Hourglass nanographenes unlock strong, robust multi-spin entanglement was earlier — Nano Letters"), 10: Case(("/category/astronomy?sort=popular", f"/article/{STAR_ARTICLE}"), - "How a single star can reshape an entire galaxy"), + "European Southern Observatory"), 11: Case((f"/article/{MAGNETIC}", f"/article/{QUANTUM_GEOMETRY}"), "Quantum geometry applied to light-based systems expands toolkit for topological photonics was published earlier"), 12: Case((f"/article/{JWST}",), @@ -283,6 +283,22 @@ def test_task_metadata_declares_all_grading_artifacts(self) -> None: self.assertIn("graphene systems", rows[9]["ques"]) self.assertIn("CO2 systems", rows[16]["ques"]) + def test_task_10_requires_a_detail_page_fact_not_the_card_title(self) -> None: + rows = [json.loads(line) for line in TASKS_FILE.read_text().splitlines() if line.strip()] + task = rows[10] + self.assertIn("open the most-viewed article", task["ques"]) + self.assertIn("Provided by", task["ques"]) + self.assertNotIn("report the title", task["ques"]) + + urls = ("/category/astronomy?sort=popular", f"/article/{STAR_ARTICLE}") + grounded = self._run(10, urls, "European Southern Observatory") + self.assert_verdict(grounded, True) + + card_only_answer = self._run( + 10, urls, "How a single star can reshape an entire galaxy" + ) + self.assert_verdict(card_only_answer, False) + def test_all_verifiers_reject_no_op(self) -> None: for task_id in range(18): with self.subTest(task=task_id): diff --git a/sites/phys_org/verify/verify_10.py b/sites/phys_org/verify/verify_10.py index c460975c..e1a19bf9 100644 --- a/sites/phys_org/verify/verify_10.py +++ b/sites/phys_org/verify/verify_10.py @@ -2,13 +2,13 @@ from verify_lib import contains_all, run_stateless, visited_category, visited_path SLUG = "how-a-single-star-can-reshape-an-entire-galaxy" -TITLE = "How a single star can reshape an entire galaxy" +PROVIDER = "European Southern Observatory" def checks(t, answer): return ([ ("nav_astronomy_popular", visited_category(t, "astronomy", "popular"), "opened Popular sort"), ("nav_top_article", visited_path(t, f"/article/{SLUG}"), "opened top article"), - ], [("answer_article_title", contains_all(answer, [TITLE]), repr(answer))]) + ], [("answer_provider", contains_all(answer, [PROVIDER]), repr(answer))]) if __name__ == "__main__": run_stateless(10, checks)