Add country, enabled, paging by date to not hit API limits. Also updated code to

.circleci/config.yml

@@ -0,0 +1,62 @@
+version: 2.1
+orbs:
+  slack: circleci/slack@3.4.2
+
+jobs:
+  build:
+    docker:
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4
+    steps:
+      - checkout
+      - run:
+          name: 'Setup virtual env'
+          command: |
+            python3 -mvenv /usr/local/share/virtualenvs/tap-appfigures
+            source /usr/local/share/virtualenvs/tap-appfigures/bin/activate
+            pip install -U pip setuptools
+            pip install .[dev]
+      - run:
+          name: 'pylint'
+          command: |
+            source /usr/local/share/virtualenvs/tap-appfigures/bin/activate
+            pylint tap_appfigures -d C,R,W
+      - add_ssh_keys
+      - run:
+          name: 'Integration Tests'
+          command: |
+            aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/sandbox dev_env.sh
+            source dev_env.sh
+            source /usr/local/share/virtualenvs/tap-tester/bin/activate
+            run-test --tap=tap-appfigures \
+                     --target=target-stitch \
+                     --orchestrator=stitch-orchestrator \
+                     --email=harrison+sandboxtest@stitchdata.com \
+                     --password=$SANDBOX_PASSWORD \
+                     --client-id=50 \
+                     tests
+      - run:
+          name: 'pylint tests'
+          command: |
+            source /usr/local/share/virtualenvs/tap-tester/bin/activate
+            pip install pylint
+            pylint tests/*.py -d 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,duplicate-code,no-name-in-module,attribute-defined-outside-init,too-many-statements,cell-var-from-loop,too-many-public-methods,missing-docstring,use-a-generator'
+      - slack/notify-on-failure:
+          only_for_branches: master
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - build:
+          context: circleci-user
+  build_daily:
+    triggers:
+      - schedule:
+          cron: "0 19 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - build:
+          context: circleci-user

.gitignore

@@ -3,5 +3,9 @@ catalog.json
 state.json
 .env
 dist/
+.idea/
 *.egg-info/
+build/
 __pycache__
+*.out
+.python-version

setup.py

@@ -9,8 +9,8 @@
       classifiers=['Programming Language :: Python :: 3 :: Only'],
       py_modules=['tap_appfigures'],
       install_requires=[
-          'singer-python==5.5.1',
-          'requests==2.20.1',
+          'singer-python==5.10.0',
+          'requests==2.25.1',
       ],
       entry_points='''
           [console_scripts]

tap_appfigures/__init__.py

@@ -21,7 +21,7 @@ def main():
     Main function - process args, build runner, execute request
     """
     args = singer.utils.parse_args(
-        required_config_keys=['api_key', 'username', 'password', 'start_date']
+        required_config_keys=['pat', 'start_date']
     )
 
     runner = AppFiguresRunner(

tap_appfigures/client.py

@@ -21,9 +21,7 @@ class AppFiguresClient:
     BASE_URI = "https://api.appfigures.com/v2/"
 
     def __init__(self, config):
-        self.api_key = config.get('api_key')
-        self.password = config.get('password')
-        self.username = config.get('username')
+        self.pat = config.get('pat')
         self.start_date = config.get('start_date')
 
     def make_request(self, uri):
@@ -32,12 +30,10 @@ def make_request(self, uri):
         and handle any errors
         """
         LOGGER.info("Making get request to {}".format(uri))
-        headers = {"X-Client-Key": self.api_key}
-        auth = (self.username, self.password)
+        headers = {"Authorization": f'Bearer {self.pat}'}
         try:
             response = requests.get(
                 self.BASE_URI + uri.lstrip("/"),
-                auth=auth,
                 headers=headers
             )
         except Exception as e:

tap_appfigures/schemas/ranks.json

@@ -23,16 +23,16 @@
                 "integer"
             ]
         },
-        "position": {
+        "positions": {
             "type": [
                 "null",
-                "integer"
+                "array"
             ]
         },
-        "delta": {
+        "deltas": {
             "type": [
                 "null",
-                "integer"
+                "array"
             ]
         }
     }

tap_appfigures/schemas/ratings.json

@@ -76,6 +76,31 @@
                 "null",
                 "number"
             ]
+        },
+        "product_id": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
+        "iso": {
+            "type": [
+                "null",
+                "string"
+            ]
+        },
+        "country": {
+            "type": [
+                "null",
+                "string"
+            ]
+        },
+        "date": {
+            "type": [
+                "null",
+                "string"
+            ],
+            "format": "date-time"
         }
     }
 }

tap_appfigures/schemas/usage.json

@@ -46,6 +46,30 @@
                 "integer"
             ]
         },
+        "app_store_views": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
+        "unique_app_store_views": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
+        "impressions": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
+        "unique_impressions": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
         "screen_views": {
             "type": [
                 "null",

tap_appfigures/streams/base.py

@@ -5,6 +5,7 @@
 import os
 
 import singer
+import datetime
 
 from tap_appfigures.utils import str_to_date, strings_to_floats, RequestError
 
@@ -28,14 +29,17 @@ class AppFiguresBase:
     URI = ''
     KEY_PROPERTIES = []
     RESPONSE_LEVELS = 2
+    ENABLED = True
 
     def __init__(self, client, state, catalog):
         self.schema = None
 
         if catalog:
             stream_details = stream_details_from_catalog(catalog, self.STREAM_NAME)
             if stream_details:
-                self.schema = stream_details.schema.to_dict()['properties']
+                if not stream_details.metadata['selected']:
+                    self.ENABLED = False
+                self.schema = stream_details.schema.to_dict()
                 self.key_properties = stream_details.key_properties
 
         if not self.schema:
@@ -64,9 +68,10 @@ def sync(self):
         These steps are the same for all streams
         Differences between streams are implemented by overriding .do_sync() method
         """
-        singer.write_schema(self.STREAM_NAME, self.schema, self.key_properties)
-        self.do_sync()
-        singer.write_state(self.state)
+        if self.ENABLED:
+            singer.write_schema(self.STREAM_NAME, self.schema, self.key_properties)
+            self.do_sync()
+            singer.write_state(self.state)
 
     @staticmethod
     def traverse_nested_dicts(dict_, levels):
@@ -92,24 +97,30 @@ def do_sync(self):
         """
         start_date = str_to_date(self.bookmark_date).strftime('%Y-%m-%d')
 
-        try:
-            response = self.client.make_request(self.URI.format(start_date))
-        except RequestError:
-            return
-
-        new_bookmark_date = self.bookmark_date
-
-        with singer.metrics.Counter('record_count', {'endpoint': self.STREAM_NAME}) as counter:
-            for entry in self.traverse_nested_dicts(response.json(), self.RESPONSE_LEVELS):
-                new_bookmark_date = max(new_bookmark_date, entry['date'])
-                entry = strings_to_floats(entry)
-                singer.write_message(singer.RecordMessage(
-                    stream=self.STREAM_NAME,
-                    record=entry,
-                ))
-            counter.increment()
-
-        self.state = singer.write_bookmark(self.state, self.STREAM_NAME, 'last_record', new_bookmark_date)
+        while str_to_date(start_date).date() < datetime.date.today():
+            end_date = min(str_to_date(start_date).date() + datetime.timedelta(days=28),
+                           datetime.date.today() - datetime.timedelta(days=1))
+
+            try:
+                response = self.client.make_request(self.URI.format(start_date, end_date.strftime('%Y-%m-%d')))
+            except RequestError:
+                return
+
+            new_bookmark_date = self.bookmark_date
+            with singer.metrics.Counter('record_count', {'endpoint': self.STREAM_NAME}) as counter:
+                for entry in self.traverse_nested_dicts(response.json(), self.RESPONSE_LEVELS):
+                    new_bookmark_date = max(new_bookmark_date, entry['date'])
+                    entry = strings_to_floats(entry)
+                    singer.write_message(singer.RecordMessage(
+                        stream=self.STREAM_NAME,
+                        record=entry,
+                    ))
+                counter.increment()
+
+            self.state = singer.write_bookmark(self.state, self.STREAM_NAME, 'last_record', new_bookmark_date)
+            if end_date == datetime.date.today() - datetime.timedelta(days=1):
+                break
+            start_date = end_date.strftime('%Y-%m-%d')
 
     def get_class_path(self):
         """

tap_appfigures/streams/products.py

@@ -14,6 +14,7 @@ def do_sync(self):
 
         product_response = self.client.make_request("/products/mine")
         product_ids = []
+
         with singer.metrics.Counter('record_count', {'endpoint': 'products'}) as counter:
 
             for product in product_response.json().values():

tap_appfigures/streams/ranks.py

@@ -13,7 +13,7 @@ class RanksStream(AppFiguresBase):
     def do_sync(self):
         start_date = str_to_date(self.bookmark_date)
         new_bookmark_date = start_date
-        product_ids = ';'.join(str(id) for id in self.product_ids)
+        product_ids = ','.join(str(id) for id in self.product_ids)
 
         while start_date.date() <= date.today():
             end_date = start_date + timedelta(days=28)

tap_appfigures/streams/ratings.py

@@ -1,7 +1,62 @@
+from collections import OrderedDict
+
 from tap_appfigures.streams.base import AppFiguresBase
+import inspect
+import os
+import itertools
+
+import singer
+import datetime
+
+from tap_appfigures.utils import str_to_date, strings_to_floats, RequestError
 
 
 class RatingsStream(AppFiguresBase):
     STREAM_NAME = 'ratings'
-    URI = '/reports/ratings?group_by=products,dates&start_date={}&granularity=daily'
-    KEY_PROPERTIES = ['product_id', 'date']
+    URI = '/reports/ratings?group_by=product,country,date&start_date={}&end_date={}&granularity=daily'
+    RESPONSE_LEVELS = 3
+    KEY_PROPERTIES = ['product_id', 'country', 'date']
+
+    def do_sync(self):
+        """
+        Main sync functionality
+        Allows for differences in schemas between catalog and the actual received data to unravel lists
+        This permits the user to get more granular ratings info (e.g. number of reviews for each rating)
+        """
+        start_date = str_to_date(self.bookmark_date).strftime('%Y-%m-%d')
+
+        while str_to_date(start_date).date() < datetime.date.today():
+            end_date = min(str_to_date(start_date).date() + datetime.timedelta(days=28),
+                           datetime.date.today() - datetime.timedelta(days=1))
+
+            try:
+                response = self.client.make_request(self.URI.format(start_date, end_date.strftime('%Y-%m-%d')))
+            except RequestError:
+                return
+
+            new_bookmark_date = self.bookmark_date
+            with singer.metrics.Counter('record_count', {'endpoint': self.STREAM_NAME}) as counter:
+                for entry in self.traverse_nested_dicts(response.json(), self.RESPONSE_LEVELS):
+                    new_bookmark_date = max(new_bookmark_date, entry['date'])
+
+                    schema_keys = [x for x in self.schema['properties'].keys() if x not in entry.keys()]
+                    entry_keys = [x for x in entry.keys() if x not in self.schema['properties'].keys() and not x.endswith('percent')]
+                    if schema_keys and entry_keys:
+                        entries = list(itertools.chain.from_iterable([entry[entry_item] for entry_item in entry_keys]))
+                        for j, schema_item in enumerate(schema_keys):
+                            entry[schema_item] = entries[j]
+                        for key in entry_keys:
+                            del(entry[key])
+
+                    entry = strings_to_floats(entry)
+
+                    singer.write_message(singer.RecordMessage(
+                        stream=self.STREAM_NAME,
+                        record=entry,
+                    ))
+                counter.increment()
+
+            self.state = singer.write_bookmark(self.state, self.STREAM_NAME, 'last_record', new_bookmark_date)
+            if end_date == datetime.date.today() - datetime.timedelta(days=1):
+                break
+            start_date = end_date.strftime('%Y-%m-%d')