diff --git a/.github/workflows/mmda-ci.yml b/.github/workflows/mmda-ci.yml index fc439211..8412ef20 100644 --- a/.github/workflows/mmda-ci.yml +++ b/.github/workflows/mmda-ci.yml @@ -11,12 +11,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.8"] + python-version: ["3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 @@ -35,10 +35,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.11' architecture: x64 - run: | sudo apt-get update @@ -53,13 +53,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.11' architecture: x64 - run: | sudo apt-get update - run: | pip install -e .[dev,svm_word_predictor] - pytest tests/test_predictors/test_svm_word_predictor.py --no-cov \ No newline at end of file + pytest tests/test_predictors/test_svm_word_predictor.py --no-cov diff --git a/pyproject.toml b/pyproject.toml index f2eb94ac..77aeac15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'mmda' -version = '0.9.18' +version = '0.9.26' description = 'MMDA - multimodal document analysis' authors = [ {name = 'Allen Institute for Artificial Intelligence', email = 'contact@allenai.org'}, @@ -9,14 +9,15 @@ license = {text = 'Apache-2.0'} readme = 'README.md' requires-python = '>=3.7' dependencies = [ - 'tqdm', - 'pdf2image', + 'tqdm<5', + 'pdf2image<2', 'pdfplumber==0.7.4', - 'requests', + 'requests<3', 'pandas<2', + 'numpy<2', 'pydantic[settings]>=1,<3', - 'ncls==0.0.66', - 'necessary>=0.3.2', + 'ncls==0.0.70', + 'necessary>=0.3.2,<1', ] [project.urls] @@ -59,107 +60,106 @@ requires = [ [project.optional-dependencies] dev = [ - 'pytest', - 'pytest-xdist', - 'pytest-cov', + 'pytest<10', + 'pytest-xdist<4', + 'pytest-cov<8', ] grobid_augment_existing_document_parser = [ 'grobid-client-python==0.0.5' ] spacy_predictors = [ - 'spacy', + 'spacy<4', ] pysbd_predictors = [ - 'pysbd', + 'pysbd<1', ] heuristic_predictors = [ - 'tokenizers', + 'tokenizers<1', 'pydantic>=1,<2', ] lp_predictors = [ - 'layoutparser', - 'torch', - 'torchvision', - 'effdet', + 'layoutparser>=0.3,<0.4', + 'torch>=2.0,<2.1', + 'torchvision>=0.15,<0.16', + 'effdet>=0.3,<0.4', 'pydantic>=1,<2' ] hf_predictors = [ - 'torch', - 'transformers', + 'torch>=2.0,<2.1', + 'transformers<6', 'smashed==0.1.10', ] vila_predictors = [ 'vila>=0.5,<0.6', - 'transformers<4.34.0', + 'transformers<4.30', 'pydantic>=1,<2', ] mention_predictor = [ - 'transformers[torch]', - 'optimum[onnxruntime]', + 'transformers[torch]<6', + 'optimum[onnxruntime]<3', 'pydantic>=1,<2', ] mention_predictor_gpu = [ - 'transformers[torch]', - 'optimum[onnxruntime-gpu]', + 'transformers[torch]<6', + 'optimum[onnxruntime-gpu]<3', 'pydantic>=1,<2', ] bibentry_predictor = [ - 'transformers', - 'unidecode', - 'torch', - 'optimum[onnxruntime]', + 'transformers<6', + 'unidecode<2', + 'torch>=2.0,<2.1', + 'optimum[onnxruntime]<3', 'pydantic>=1,<2', ] bibentry_predictor_gpu = [ - 'transformers', - 'unidecode', - 'torch', - 'optimum[onnxruntime-gpu]', + 'transformers<6', + 'unidecode<2', + 'torch>=2.0,<2.1', + 'optimum[onnxruntime-gpu]<3', 'pydantic>=1,<2' ] bibentry_detection_predictor = [ - 'Pillow<10', - 'layoutparser', - 'torch==1.8.0+cu111', - 'torchvision==0.9.0+cu111', - 'pydantic<=1,<2', + 'Pillow>=9.3,<10', + 'layoutparser>=0.3,<0.4', + 'pydantic>=1,<2', ] citation_links = [ - 'numpy', - 'thefuzz[speedup]', - 'scikit-learn', - 'xgboost', + 'numpy<2', + 'thefuzz[speedup]<1', + 'scikit-learn<1.3', + 'xgboost<2', 'pydantic>=1,<2', ] section_nesting = [ - 'numpy', - 'scipy', - 'xgboost', + 'numpy<2', + 'scipy<2', + 'scikit-learn<1.3', + 'xgboost<2', ] figure_table_predictors = [ - 'scipy', + 'scipy<2', 'pydantic>=1,<2', ] svm_word_predictor = [ - 'scikit-learn', - 'scipy', - 'numpy', - 'tokenizers', + 'scikit-learn<1.3', + 'scipy<2', + 'numpy<2', + 'tokenizers<1', 'pydantic>=1,<2', ] recipes = [ - 'layoutparser', - 'transformers', - 'torch', - 'torchvision', - 'effdet', + 'layoutparser>=0.3,<0.4', + 'transformers<6', + 'torch>=2.0,<2.1', + 'torchvision>=0.15,<0.16', + 'effdet>=0.3,<0.4', 'vila>=0.5,<0.6', 'smashed==0.1.10', - 'scikit-learn', - 'scipy', - 'numpy', - 'tokenizers', - 'xgboost', + 'scikit-learn<1.3', + 'scipy<2', + 'numpy<2', + 'tokenizers<1', + 'xgboost<2', ] @@ -171,7 +171,7 @@ pythonpath = [ ] python_classes = ['Test*', '*Test'] log_format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s' -log_level = 'DEBUG' +log_level = 'WARNING' markers = [] filterwarnings = [] diff --git a/src/ai2_internal/config.yaml b/src/ai2_internal/config.yaml index 7b4dff6c..92490210 100644 --- a/src/ai2_internal/config.yaml +++ b/src/ai2_internal/config.yaml @@ -24,7 +24,7 @@ model_variants: artifacts_s3_path: s3://ai2-timo-registry/model-artifacts/vila/ivila-row-layoutlm-finetuned-s2vl-v2.tar.gz # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: 3.8 + python_version: "3.11" cuda: True @@ -51,7 +51,7 @@ model_variants: extras_require: ["dev", "lp_predictors"] # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: 3.8 + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: True @@ -88,7 +88,7 @@ model_variants: artifacts_s3_path: s3://ai2-timo-registry/model-artifacts/bibentryparser/v0.onnx.tar.gz # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: 3.8 + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: False @@ -125,7 +125,7 @@ model_variants: artifacts_s3_path: s3://ai2-timo-registry/model-artifacts/bibentryparser/v0.onnx.tar.gz # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: 3.8 + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: True @@ -163,7 +163,7 @@ model_variants: artifacts_s3_path: s3://ai2-s2-mmda/models/citation-mentions/2022-07-27-minilm-10k/model/artifacts-onnx16.tar.gz # Version of python required for model runtime - python_version: "3.8" + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: True @@ -194,7 +194,7 @@ model_variants: artifacts_s3_path: s3://ai2-timo-registry/model-artifacts/citationlinks/v1.tar.gz # Version of python required for model runtime - python_version: "3.8" + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: False @@ -222,14 +222,14 @@ model_variants: artifacts_s3_path: s3://ai2-s2-analytics/danyh/extract-stuff-from-pdfs/artifacts/anchor/archive.tar.gz # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: "3.8" + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: true - # One of the versions here: https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md#ubuntu2004, but less than 11.4.3. - # If cuda=True and cuda_version is unspecified, defaults to 11.4.2. - cuda_version: "11.1.1" + # One of the versions here: https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md#ubuntu2204 + # If cuda=True and cuda_version is unspecified, defaults to 11.8.0. + cuda_version: "11.8.0" # Python path to a fn in == that # returns a unittest.TestCase. Builder function receives a model container @@ -243,9 +243,9 @@ model_variants: # pip installing dependencies listed in setup.py so that detectron2 installs successfully docker_run_commands: [ "apt-get update && apt-get install -y poppler-utils libgl1", - "pip install layoutparser", - "pip install torch==1.8.0+cu111 torchvision==0.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html", - "pip install 'detectron2@git+https://github.com/facebookresearch/detectron2.git@v0.6#egg=detectron2'"] + "pip install torch==2.4.1+cu118 torchvision==0.19.1+cu118 --index-url https://download.pytorch.org/whl/cu118", + "pip install https://github.com/miropsota/torch_packages_builder/releases/download/detectron2-0.6%2Bfd27788/detectron2-0.6%2Bfd27788pt2.4.1cu118-cp311-cp311-linux_x86_64.whl", + "pip install layoutparser"] # Any additional sets of dependencies required by the model. # These are the 'extras_require' keys in your setup.py. @@ -267,7 +267,7 @@ model_variants: integration_test: ai2_internal.figure_table_predictors.integration_test.TestInterfaceIntegration # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: 3.8 + python_version: "3.11" cuda: False @@ -305,7 +305,7 @@ model_variants: artifacts_s3_path: null # Version of python required for model runtime, e.g. 3.7, 3.8, 3.9 - python_version: 3.8 + python_version: "3.11" # Whether this model supports CUDA GPU acceleration cuda: false