From 63c6d5a8894ebeca27c9203665cc86cd62b63582 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Wed, 11 Dec 2019 23:14:39 -0700 Subject: [PATCH 1/9] Add unit tests for es log object parsing --- packages/api/es/indexer.js | 83 +++++++++++-------- .../api/tests/serial/es/test-es-indexer.js | 51 +++++++++++- 2 files changed, 99 insertions(+), 35 deletions(-) diff --git a/packages/api/es/indexer.js b/packages/api/es/indexer.js index 498e86f2242b..96d955f11aaf 100644 --- a/packages/api/es/indexer.js +++ b/packages/api/es/indexer.js @@ -39,6 +39,50 @@ async function createIndex(esClient, indexName) { log.info(`Created esIndex ${indexName}`); } + +/** + * Parses a StepFunction log payload and returns a es logsrecord object + * + * @param {Object} payload - Stepfunction log payload + * @returns {Object} - ElasticSearch log record + */ +function parsePayload(payload) { + let record; + try { + // cumulus log message has extra aws messages before the json message, + // only the json message should be logged to elasticsearch. + // example message: + // 2018-06-0 1T17:45:27.108Z a714a0ef-f141-4e52-9661-58ca2233959a + // {"level": "info", "timestamp": "2018-06-01T17:45:27.108Z", + // "message": "uploaded s3://bucket/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf.met"} + const entryParts = payload.message.trim().split('\t'); + // cumulus log message + const messageStartIndex = entryParts.findIndex((e) => e.startsWith('{')); + if (entryParts.length >= 3 && messageStartIndex + && entryParts[entryParts.length - 1].endsWith('}')) { + record = JSON.parse(entryParts.slice(messageStartIndex).join('\t')); + record.RequestId = entryParts[1]; + } else { // other logs e.g. cumulus-ecs-task + record = JSON.parse(payload.message); + } + // level is number in elasticsearch + if (isString(record.level)) record.level = log.convertLogLevel(record.level); + } catch (e) { + record = { + message: payload.message.trim(), + sender: payload.sender, + executions: payload.executions, + timestamp: payload.timestamp, + version: payload.version, + level: 30, + pid: 1, + name: 'cumulus' + }; + } + return record; +} + + /** * Extracts info from a stepFunction message and indexes it to * an ElasticSearch @@ -52,40 +96,10 @@ async function createIndex(esClient, indexName) { async function indexLog(esClient, payloads, index = defaultIndexAlias, type = 'logs') { const body = []; - payloads.forEach((p) => { - body.push({ index: { _index: index, _type: type, _id: p.id } }); - let record; - try { - // cumulus log message has extra aws messages before the json message, - // only the json message should be logged to elasticsearch. - // example message: - // 2018-06-01T17:45:27.108Z a714a0ef-f141-4e52-9661-58ca2233959a - // {"level": "info", "timestamp": "2018-06-01T17:45:27.108Z", - // "message": "uploaded s3://bucket/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf.met"} - const entryParts = p.message.trim().split('\t'); - // cumulus log message - if (entryParts.length >= 3 && entryParts[2].startsWith('{') - && entryParts[entryParts.length - 1].endsWith('}')) { - record = JSON.parse(entryParts.slice(2).join('\t')); - record.RequestId = entryParts[1]; - } else { // other logs e.g. cumulus-ecs-task - record = JSON.parse(p.message); - } - // level is number in elasticsearch - if (isString(record.level)) record.level = log.convertLogLevel(record.level); - } catch (e) { - record = { - message: p.message.trim(), - sender: p.sender, - executions: p.executions, - timestamp: p.timestamp, - version: p.version, - level: 30, - pid: 1, - name: 'cumulus' - }; - } - body.push(record); + payloads.forEach((payload) => { + body.push({ index: { _index: index, _type: type, _id: payload.id } }); + const parsedPayload = parsePayload(payload); + body.push(parsedPayload); }); const actualEsClient = esClient || (await Search.es()); @@ -176,6 +190,7 @@ function indexProvider(esClient, payload, index = defaultIndexAlias, type = 'pro * @param {string} type - Elasticsearch type (default: rule) * @returns {Promise} Elasticsearch response */ + function indexRule(esClient, payload, index = defaultIndexAlias, type = 'rule') { return genericRecordUpdate(esClient, payload.name, payload, index, type); } diff --git a/packages/api/tests/serial/es/test-es-indexer.js b/packages/api/tests/serial/es/test-es-indexer.js index 58c81b30a048..fa85bbc82623 100644 --- a/packages/api/tests/serial/es/test-es-indexer.js +++ b/packages/api/tests/serial/es/test-es-indexer.js @@ -2,6 +2,7 @@ const test = require('ava'); const sinon = require('sinon'); +const rewire = require('rewire'); const fs = require('fs'); const path = require('path'); const aws = require('@cumulus/common/aws'); @@ -11,7 +12,7 @@ const { constructCollectionId } = require('@cumulus/common/collection-config-sto const { noop } = require('@cumulus/common/util'); const StepFunctions = require('@cumulus/common/StepFunctions'); -const indexer = require('../../../es/indexer'); +const indexer = rewire('../../../es/indexer'); const { Search } = require('../../../es/search'); const models = require('../../../models'); const { fakeGranuleFactory, fakeCollectionFactory, deleteAliases } = require('../../../lib/testUtils'); @@ -559,3 +560,51 @@ test.serial('Create new index - index already exists', async (t) => { await esClient.indices.delete({ index: newIndex }); }); + +test.serial('parsePayload correctly parses AWS Linux style console output', async (t) => { + const parsePayload = indexer.__get__('parsePayload'); + const expected = { + some: 'key', + sender: 'some sender', + message: 'a messaage', + RequestId: 'a714a0ef-f141-4e52-9661-58ca2233959a' + }; + const actual = parsePayload({ sender: 'fixture_sender', message: '2018-06-01T17:45:27.108Z\ta714a0ef-f141-4e52-9661-58ca2233959a\t{"some": "key", "sender": "some sender", "message": "a messaage"}' }); + t.deepEqual(actual, expected); +}); + +test.serial('parsePayload correctly parses AWS Linux 2 style console output', async (t) => { + const parsePayload = indexer.__get__('parsePayload'); + const expected = { + some: 'key', + sender: 'some sender', + message: 'a messaage', + RequestId: 'a714a0ef-f141-4e52-9661-58ca2233959a' + }; + const actual = parsePayload({ sender: 'fixture_sender', message: '2018-06-01T17:45:27.108Z\ta714a0ef-f141-4e52-9661-58ca2233959a\tINFO\t{"some": "key", "sender": "some sender", "message": "a messaage"}' }); + t.deepEqual(actual, expected); +}); + + +test.serial('parsePayload correctly handles unparseable record', async (t) => { + const parsePayload = indexer.__get__('parsePayload'); + const payload = { + message: 'INFO MESSAGE', + sender: 'AWS sender', + executions: 'some execution value', + timestamp: '2018-06-01T17:45:27.108Z', + version: '1' + }; + const expected = { + message: 'INFO MESSAGE', + sender: 'AWS sender', + executions: 'some execution value', + timestamp: '2018-06-01T17:45:27.108Z', + version: '1', + level: 30, + pid: 1, + name: 'cumulus' + }; + const actual = parsePayload(payload); + t.deepEqual(actual, expected); +}); From de9c03710aa1081bba60a6cef15b6679e9d481f7 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Thu, 12 Dec 2019 21:59:53 -0700 Subject: [PATCH 2/9] Backport fix for CUMULUS-1626/node 10 to 1.14.x baseline --- bamboo/docker-compose.yml | 2 +- example/app/config.yml | 2 +- example/package.json | 5 +++-- package.json | 5 +++-- packages/api/ecs/async-operation/package.json | 2 +- packages/api/package.json | 2 +- packages/common/package.json | 4 ++-- packages/deployment/package.json | 2 +- packages/ingest/package.json | 2 +- packages/integration-tests/package.json | 2 +- tf-modules/s3-replicator/package.json | 2 +- 11 files changed, 16 insertions(+), 14 deletions(-) diff --git a/bamboo/docker-compose.yml b/bamboo/docker-compose.yml index 08609c062c35..d4c28cd753c5 100644 --- a/bamboo/docker-compose.yml +++ b/bamboo/docker-compose.yml @@ -37,7 +37,7 @@ services: environment: SERVICES: 'kinesis,lambda,s3,sns,sqs,dynamodb,dynamodbstreams,cloudwatch,cloudwatchlogs' build_env: - image: jlkovarik/cumulus_build_env:1 + image: jlkovarik/cumulus_build_env:2 volumes: - ../:/source/cumulus environment: diff --git a/example/app/config.yml b/example/app/config.yml index f49625d373b7..4e4a7fa7cbac 100644 --- a/example/app/config.yml +++ b/example/app/config.yml @@ -61,7 +61,7 @@ default: # test configuration ends cmaDir: '/opt/' - sandbox_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:6 + sandbox_cma_layer: arn:aws:lambda:us-east-1:596205514787:layer:Cumulus_Message_Adapter_test:16 sit_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:2 cma_layer: '{{sandbox_cma_layer}}' prefix: 'test-cumulus-integration' diff --git a/example/package.json b/example/package.json index 6fcb666f1dde..75139d0a723f 100644 --- a/example/package.json +++ b/example/package.json @@ -55,11 +55,12 @@ "@cumulus/sf-sns-report": "1.14.4", "@cumulus/sync-granule": "1.14.4", "@cumulus/test-processing": "1.14.4", - "aws-sdk": "^2.227.1", + "aws-sdk": "^2.585.0", "child-process-promise": "^2.2.1", "lodash.differencewith": "^4.5.0", "lodash.isequal": "^4.5.0", - "p-retry": "^2.0.0" + "p-retry": "^2.0.0", + "ssh2-streams": "^0.4.8" }, "devDependencies": { "@cumulus/test-data": "1.14.4", diff --git a/package.json b/package.json index 955f3f0033d3..d2de1b8fd7be 100644 --- a/package.json +++ b/package.json @@ -71,10 +71,11 @@ "eslint-plugin-unicorn": "^4.0.3", "lerna": "^3.13.2", "nyc": "^14.0.0", - "simple-git": "^1.96.0" + "simple-git": "^1.96.0", + "ssh2-streams": "^0.4.8" }, "dependencies": { - "aws-sdk": "^2.238.1", + "aws-sdk": "^2.585.0", "fs-extra": "^5.0.0", "latest-version": "^4.0.0", "semver": "^5.5.0" diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index 69976d4ba70f..f520c98feb12 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -4,7 +4,7 @@ }, "dependencies": { "@cumulus/logger": "^1.13.0", - "aws-sdk": "^2.279.1", + "aws-sdk": "^2.585.0", "got": "^9.2.2", "lodash.iserror": "^3.1.1", "p-retry": "^2.0.0" diff --git a/packages/api/package.json b/packages/api/package.json index 52f31900641d..8e725586f636 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -58,7 +58,7 @@ "ajv": "^5.2.2", "archiver": "^2.1.1", "aws-elasticsearch-connector": "8.1.3", - "aws-sdk": "^2.238.1", + "aws-sdk": "^2.585.0", "aws-serverless-express": "^3.3.5", "basic-auth": "^1.1.0", "body-parser": "^1.18.3", diff --git a/packages/common/package.json b/packages/common/package.json index 8810b060b8c0..a80b68350625 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -47,7 +47,7 @@ "@cumulus/logger": "1.14.4", "ajv": "^5.2.2", "async": "^2.0.0", - "aws-sdk": "^2.250.1", + "aws-sdk": "^2.585.0", "expect.js": "^0.3.1", "follow-redirects": "^1.2.4", "formidable": "^1.2.1", @@ -82,7 +82,7 @@ "p-retry": "^2.0.0", "pump": "^3.0.0", "randexp": "^0.4.9", - "ssh2": "^0.5.5", + "ssh2": "^0.8.7", "url-join": "^4.0.0", "uuid": "^3.2.1" }, diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 095511f0cf21..4446d9e1b9ff 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -43,7 +43,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "aws-sdk": "^2.479.0", + "aws-sdk": "^2.585.0", "commander": "^2.20.0", "extract-zip": "^1.6.6", "fs-extra": "^5.0.0", diff --git a/packages/ingest/package.json b/packages/ingest/package.json index a5a761232698..32913f1ead3c 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -39,7 +39,7 @@ "@cumulus/cmrjs": "1.14.4", "@cumulus/common": "1.14.4", "@cumulus/pvl": "1.14.4", - "aws-sdk": "^2.238.1", + "aws-sdk": "^2.585.0", "cksum": "^1.3.0", "encodeurl": "^1.0.2", "fs-extra": "^5.0.0", diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 3cc29257c922..5d4467c32bc6 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -28,7 +28,7 @@ "@cumulus/cmrjs": "1.14.4", "@cumulus/common": "1.14.4", "@cumulus/deployment": "1.14.4", - "aws-sdk": "^2.238.1", + "aws-sdk": "^2.585.0", "base-64": "^0.1.0", "commander": "^2.15.0", "fs-extra": "^5.0.0", diff --git a/tf-modules/s3-replicator/package.json b/tf-modules/s3-replicator/package.json index 237412f2d931..af2d7e1b5a77 100644 --- a/tf-modules/s3-replicator/package.json +++ b/tf-modules/s3-replicator/package.json @@ -12,7 +12,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "aws-sdk": "^2.487.0" + "aws-sdk": "^2.585.0" }, "devDependencies": { "ava": "^2.1.0" From a084ba39f04f2a36b242ba1452ea351f9948456e Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Thu, 12 Dec 2019 22:05:25 -0700 Subject: [PATCH 3/9] Update package.json --- .nvmrc | 2 +- example/package.json | 2 +- package.json | 2 +- packages/api/ecs/async-operation/package.json | 2 +- packages/api/package.json | 2 +- packages/checksum/package.json | 2 +- packages/cmr-client/package.json | 2 +- packages/cmrjs/package.json | 2 +- packages/common/package.json | 2 +- packages/deployment/package.json | 2 +- packages/ingest/package.json | 2 +- packages/integration-tests/package.json | 2 +- packages/logger/package.json | 2 +- packages/task-debug/package.json | 2 +- packages/test-data/package.json | 2 +- tasks/discover-granules/package.json | 2 +- tasks/discover-pdrs/package.json | 2 +- tasks/files-to-granules/package.json | 2 +- tasks/hello-world/package.json | 2 +- tasks/move-granules/package.json | 2 +- tasks/parse-pdr/package.json | 2 +- tasks/pdr-status-check/package.json | 2 +- tasks/post-to-cmr/package.json | 2 +- tasks/queue-granules/package.json | 2 +- tasks/queue-pdrs/package.json | 2 +- tasks/sf-sns-report/package.json | 2 +- tasks/sync-granule/package.json | 2 +- tasks/test-processing/package.json | 2 +- tf-modules/distribution/package.json | 2 +- tf-modules/report-executions/package.json | 2 +- tf-modules/report-granules/package.json | 2 +- tf-modules/report-pdrs/package.json | 2 +- tf-modules/s3-replicator/package.json | 2 +- 33 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.nvmrc b/.nvmrc index 22ec1e649434..e76d1606bf62 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -8.10 +10.16.4 diff --git a/example/package.json b/example/package.json index 75139d0a723f..87fabb6f3c73 100644 --- a/example/package.json +++ b/example/package.json @@ -5,7 +5,7 @@ "private": true, "main": "index.js", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "jasmine && npm run parallel-tests", diff --git a/package.json b/package.json index d2de1b8fd7be..ee5cb65c3f66 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "bin": { "build-tasks-doc": "./bin/build-tasks-doc.js" diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index f520c98feb12..7797a9f0c11b 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -1,6 +1,6 @@ { "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "dependencies": { "@cumulus/logger": "^1.13.0", diff --git a/packages/api/package.json b/packages/api/package.json index 8e725586f636..d056951c1153 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -4,7 +4,7 @@ "description": "Lambda functions for handling all daac's API operations", "main": "index.js", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "bin": { "cumulus-api": "./bin/cli.js" diff --git a/packages/checksum/package.json b/packages/checksum/package.json index 92bb43393ce9..f142ca0a8778 100644 --- a/packages/checksum/package.json +++ b/packages/checksum/package.json @@ -3,7 +3,7 @@ "version": "1.14.4", "description": "Cumulus checksum utilities", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/packages/cmr-client/package.json b/packages/cmr-client/package.json index 8c635add7746..5e42415286cc 100644 --- a/packages/cmr-client/package.json +++ b/packages/cmr-client/package.json @@ -2,7 +2,7 @@ "name": "@cumulus/cmr-client", "version": "1.14.4", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "build-docs": "jsdoc2md --heading-depth 3 --template templates/README.hbs CMR.js CMRSearchConceptQueue.js > README.md", diff --git a/packages/cmrjs/package.json b/packages/cmrjs/package.json index 51ecb3bcf75b..0b4787800785 100644 --- a/packages/cmrjs/package.json +++ b/packages/cmrjs/package.json @@ -3,7 +3,7 @@ "version": "1.14.4", "description": "A node SDK for CMR", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/packages/common/package.json b/packages/common/package.json index a80b68350625..d989408037d4 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -8,7 +8,7 @@ "NASA" ], "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "publishConfig": { "access": "public" diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 4446d9e1b9ff..bb8646c3976b 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -16,7 +16,7 @@ "files": "test" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "keywords": [ "CUMULUS" diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 32913f1ead3c..1634ce173cb5 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -3,7 +3,7 @@ "version": "1.14.4", "description": "Ingest utilities", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 5d4467c32bc6..ed16c54c6a25 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -6,7 +6,7 @@ "cumulus-test": "./bin/cli.js" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "main": "index.js", "publishConfig": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 49328ea36b56..97399f159c3d 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -8,7 +8,7 @@ "NASA" ], "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "publishConfig": { "access": "public" diff --git a/packages/task-debug/package.json b/packages/task-debug/package.json index eb66766ff825..a95585b14e84 100644 --- a/packages/task-debug/package.json +++ b/packages/task-debug/package.json @@ -10,7 +10,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "author": "Cumulus Authors", "license": "Apache-2.0", diff --git a/packages/test-data/package.json b/packages/test-data/package.json index 15e4a7088ada..84152394c2b7 100644 --- a/packages/test-data/package.json +++ b/packages/test-data/package.json @@ -16,7 +16,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "author": "Cumulus Authors", "license": "Apache-2.0" diff --git a/tasks/discover-granules/package.json b/tasks/discover-granules/package.json index ff8586484158..121e6110eea7 100644 --- a/tasks/discover-granules/package.json +++ b/tasks/discover-granules/package.json @@ -12,7 +12,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/discover-pdrs/package.json b/tasks/discover-pdrs/package.json index 2052a6e69f01..44f4f410e0dd 100644 --- a/tasks/discover-pdrs/package.json +++ b/tasks/discover-pdrs/package.json @@ -12,7 +12,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/files-to-granules/package.json b/tasks/files-to-granules/package.json index 0fb62d2bc5a5..41e27b1857ae 100644 --- a/tasks/files-to-granules/package.json +++ b/tasks/files-to-granules/package.json @@ -22,7 +22,7 @@ "prepare": "npm run build" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "nyc": { "exclude": [ diff --git a/tasks/hello-world/package.json b/tasks/hello-world/package.json index 6b7cddb3c011..7765137d14d3 100644 --- a/tasks/hello-world/package.json +++ b/tasks/hello-world/package.json @@ -12,7 +12,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/move-granules/package.json b/tasks/move-granules/package.json index 9cfcc6b15170..f3be0050dc10 100644 --- a/tasks/move-granules/package.json +++ b/tasks/move-granules/package.json @@ -15,7 +15,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/parse-pdr/package.json b/tasks/parse-pdr/package.json index 99403a54a913..99116843f535 100644 --- a/tasks/parse-pdr/package.json +++ b/tasks/parse-pdr/package.json @@ -16,7 +16,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/pdr-status-check/package.json b/tasks/pdr-status-check/package.json index 8127b8c35161..96005aa335df 100644 --- a/tasks/pdr-status-check/package.json +++ b/tasks/pdr-status-check/package.json @@ -10,7 +10,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/post-to-cmr/package.json b/tasks/post-to-cmr/package.json index bb9090795c43..c3c52ba807a1 100644 --- a/tasks/post-to-cmr/package.json +++ b/tasks/post-to-cmr/package.json @@ -15,7 +15,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/queue-granules/package.json b/tasks/queue-granules/package.json index bcc1ff0f5e4a..5cf854caf73a 100644 --- a/tasks/queue-granules/package.json +++ b/tasks/queue-granules/package.json @@ -15,7 +15,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/queue-pdrs/package.json b/tasks/queue-pdrs/package.json index 1a733ee755e4..ca1d436baafc 100644 --- a/tasks/queue-pdrs/package.json +++ b/tasks/queue-pdrs/package.json @@ -15,7 +15,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/sf-sns-report/package.json b/tasks/sf-sns-report/package.json index 41efa92f5252..14a3fb64e83e 100644 --- a/tasks/sf-sns-report/package.json +++ b/tasks/sf-sns-report/package.json @@ -15,7 +15,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/sync-granule/package.json b/tasks/sync-granule/package.json index 18779f168710..27bca27ad8c4 100644 --- a/tasks/sync-granule/package.json +++ b/tasks/sync-granule/package.json @@ -15,7 +15,7 @@ "access": "public" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava", diff --git a/tasks/test-processing/package.json b/tasks/test-processing/package.json index cdda6a92aa66..2ece25d38b93 100644 --- a/tasks/test-processing/package.json +++ b/tasks/test-processing/package.json @@ -9,7 +9,7 @@ "url": "https://github.com/nasa/cumulus" }, "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "build": "webpack", diff --git a/tf-modules/distribution/package.json b/tf-modules/distribution/package.json index 6ad379e9e019..83f2bbc5ae75 100644 --- a/tf-modules/distribution/package.json +++ b/tf-modules/distribution/package.json @@ -3,7 +3,7 @@ "private": true, "version": "1.14.4", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "build": "webpack && (cd dist && rm -f package.zip && zip src.zip index.js)", diff --git a/tf-modules/report-executions/package.json b/tf-modules/report-executions/package.json index 5d4b3faa8cd7..c689a2b57667 100644 --- a/tf-modules/report-executions/package.json +++ b/tf-modules/report-executions/package.json @@ -5,7 +5,7 @@ "description": "Store workflow execution information to database", "main": "index.js", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "build": "rm -rf dist && mkdir dist", diff --git a/tf-modules/report-granules/package.json b/tf-modules/report-granules/package.json index f1ce16d2491f..5986f76454ad 100644 --- a/tf-modules/report-granules/package.json +++ b/tf-modules/report-granules/package.json @@ -4,7 +4,7 @@ "description": "Cumulus Workflow Granule Reporting Module", "main": "index.js", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "build": "rm -rf dist && mkdir dist", diff --git a/tf-modules/report-pdrs/package.json b/tf-modules/report-pdrs/package.json index c9da3ea6ce7b..bb4b7d8004d2 100644 --- a/tf-modules/report-pdrs/package.json +++ b/tf-modules/report-pdrs/package.json @@ -4,7 +4,7 @@ "description": "Cumulus Workflow PDR Reporting Module", "main": "index.js", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "build": "rm -rf dist && mkdir dist", diff --git a/tf-modules/s3-replicator/package.json b/tf-modules/s3-replicator/package.json index af2d7e1b5a77..917412cdf709 100644 --- a/tf-modules/s3-replicator/package.json +++ b/tf-modules/s3-replicator/package.json @@ -4,7 +4,7 @@ "description": "Replicate S3 Events to alternate bucket. Solves same-region replication.", "main": "index.js", "engines": { - "node": ">=8.10.0" + "node": ">=10.16.3" }, "scripts": { "test": "ava" From da3cff6d34cc4a608e15761666cc342ae8c0661b Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Mon, 16 Dec 2019 22:14:46 -0700 Subject: [PATCH 4/9] Additional backports from PR --- .nvmrc | 2 +- docs/deployment/README.md | 2 +- example/README.md | 2 +- packages/api/ecs/async-operation/Dockerfile | 2 +- .../app/cloudformation.template.yml | 2 +- .../deployment/app/cumulus_api.template.yml | 2 +- packages/ingest/package.json | 2 +- packages/task-debug/README.md | 2 +- tf-modules/archive/api.tf | 2 +- tf-modules/archive/bootstrap.tf | 2 +- tf-modules/archive/bulk_delete.tf | 2 +- tf-modules/archive/clean_executions.tf | 2 +- tf-modules/archive/db_indexer.tf | 2 +- tf-modules/archive/ems_reporting.tf | 6 ++--- tf-modules/archive/execute_migrations.tf | 2 +- tf-modules/archive/index_from_database.tf | 2 +- tf-modules/archive/log2elasticsearch.tf | 2 +- tf-modules/archive/reconciliation_report.tf | 2 +- tf-modules/archive/sns2elasticsearch.tf | 2 +- tf-modules/distribution/main.tf | 2 +- tf-modules/ingest/lambda-functions.tf | 22 +++++++++---------- tf-modules/report-executions/main.tf | 2 +- tf-modules/report-granules/main.tf | 2 +- tf-modules/report-pdrs/main.tf | 2 +- 24 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.nvmrc b/.nvmrc index e76d1606bf62..d19159826d17 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -10.16.4 +10.16.3 diff --git a/docs/deployment/README.md b/docs/deployment/README.md index c32e5a564dc7..d6a5f3be6b82 100644 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -35,7 +35,7 @@ The process involves: ### Linux/MacOS software requirements * git -* [node 8.10](https://nodejs.org/en/) (use [nvm](https://github.com/creationix/nvm) to upgrade/downgrade) +* [node 10.16.3](https://nodejs.org/en/) (use [nvm](https://github.com/creationix/nvm) to upgrade/downgrade) * [npm](https://www.npmjs.com/get-npm) * sha1sum or md5sha1sum * zip diff --git a/example/README.md b/example/README.md index 8c081a34c301..a74b1e5a95ed 100644 --- a/example/README.md +++ b/example/README.md @@ -172,7 +172,7 @@ https://abc123.execute-api.us-east-1.amazonaws.com:7000/DEV/my-protected/path/to An S3 Access lambda is needed in the us-west-2 region to run the integration tests. To initially create the lambda, run: ``` -aws lambda create-function --region us-west-2 --function-name -S3AccessTest --zip-file fileb://app/build/cloudformation/-S3AccessTest.zip --role arn:aws:iam:::role/-lambda-processing --handler index.handler --runtime nodejs8.10 --profile +aws lambda create-function --region us-west-2 --function-name -S3AccessTest --zip-file fileb://app/build/cloudformation/-S3AccessTest.zip --role arn:aws:iam:::role/-lambda-processing --handler index.handler --runtime nodejs10.x --profile ``` Replace `` with your account Id, `` with your stack name, `` with your iam prefix name, and use your NGAP profile. An S3AccessTest zip file `` can be found under `app/build/` following a stack deployment (either in `cloudformation` or `workflow_lambda_versions` depending on your stack configuration). The version of the zip file that you upload does not matter, but you need to deploy something to us-west-2 so that Bamboo can update it with current lambdas. diff --git a/packages/api/ecs/async-operation/Dockerfile b/packages/api/ecs/async-operation/Dockerfile index 768cb33850f5..43049fef0b75 100644 --- a/packages/api/ecs/async-operation/Dockerfile +++ b/packages/api/ecs/async-operation/Dockerfile @@ -1,4 +1,4 @@ -FROM node:8.10 +FROM node:10.16.3 USER root RUN sed -i -e '/jessie-updates/d' /etc/apt/sources.list diff --git a/packages/deployment/app/cloudformation.template.yml b/packages/deployment/app/cloudformation.template.yml index 630e0971e855..e56befd14462 100644 --- a/packages/deployment/app/cloudformation.template.yml +++ b/packages/deployment/app/cloudformation.template.yml @@ -519,7 +519,7 @@ Resources: Role: {{../iams.lambdaProcessingRoleArn}} {{/ifNotEquals}} {{/if}} - Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs8.10{{/if}} + Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs10.x{{/if}} Timeout: {{this.timeout}} {{# if this.deadletterqueue}} DeadLetterConfig: diff --git a/packages/deployment/app/cumulus_api.template.yml b/packages/deployment/app/cumulus_api.template.yml index 30a88500a480..8885700b1fa5 100644 --- a/packages/deployment/app/cumulus_api.template.yml +++ b/packages/deployment/app/cumulus_api.template.yml @@ -359,7 +359,7 @@ Resources: Role: {{../parent.iams.lambdaProcessingRoleArn}} {{/ifNotEquals}} {{/if}} - Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs8.10{{/if}} + Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs10.x{{/if}} Timeout: {{this.timeout}} Tags: - Key: Project diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 1634ce173cb5..603648b026cc 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -45,7 +45,7 @@ "fs-extra": "^5.0.0", "got": "^9.2.1", "is-ip": "^2.0.0", - "jsftp": "https://github.com/yjpa7145/jsftp.git#Fix-partial-file-issue", + "jsftp": "https://github.com/jkovarik/jsftp.git#add_288", "json-loader": "~0.5.7", "lodash.clonedeep": "^4.5.0", "lodash.flatten": "^4.4.0", diff --git a/packages/task-debug/README.md b/packages/task-debug/README.md index 1fb1b80b997a..e73d8c4ae9b7 100644 --- a/packages/task-debug/README.md +++ b/packages/task-debug/README.md @@ -38,7 +38,7 @@ is always used. * Currently only linear workflows are supported, e.g., no branching, but it should be pretty simple to add branching support later. -* Because of the dynamic loading of tasks it is recommend to use node 8 without transpiling when +* Because of the dynamic loading of tasks it is recommend to use node 10 without transpiling when debugging to avoid the need to generate and deal with source maps ## Contributing diff --git a/tf-modules/archive/api.tf b/tf-modules/archive/api.tf index c276d4ce29a6..26330e8ae0b9 100644 --- a/tf-modules/archive/api.tf +++ b/tf-modules/archive/api.tf @@ -16,7 +16,7 @@ resource "aws_lambda_function" "api" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/app/lambda.zip") handler = "index.handler" role = aws_iam_role.lambda_api_gateway.arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 environment { variables = { diff --git a/tf-modules/archive/bootstrap.tf b/tf-modules/archive/bootstrap.tf index c636b4eb323e..f4c79e9d2f81 100644 --- a/tf-modules/archive/bootstrap.tf +++ b/tf-modules/archive/bootstrap.tf @@ -12,7 +12,7 @@ resource "aws_lambda_function" "custom_bootstrap" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/bootstrap/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 320 environment { diff --git a/tf-modules/archive/bulk_delete.tf b/tf-modules/archive/bulk_delete.tf index 0a6f7899d2cf..8363f4ee7f61 100644 --- a/tf-modules/archive/bulk_delete.tf +++ b/tf-modules/archive/bulk_delete.tf @@ -4,7 +4,7 @@ resource "aws_lambda_function" "bulk_delete" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/bulkDelete/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" memory_size = 1024 timeout = 300 environment { diff --git a/tf-modules/archive/clean_executions.tf b/tf-modules/archive/clean_executions.tf index b830f0e729de..8d3c3dea65a9 100644 --- a/tf-modules/archive/clean_executions.tf +++ b/tf-modules/archive/clean_executions.tf @@ -11,7 +11,7 @@ resource "aws_lambda_function" "clean_executions" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/cleanExecutions/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 900 memory_size = 192 dead_letter_config { diff --git a/tf-modules/archive/db_indexer.tf b/tf-modules/archive/db_indexer.tf index 2a6ae49491eb..1cd4ebabfec9 100644 --- a/tf-modules/archive/db_indexer.tf +++ b/tf-modules/archive/db_indexer.tf @@ -11,7 +11,7 @@ resource "aws_lambda_function" "db_indexer" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/dbIndexer/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 320 dead_letter_config { diff --git a/tf-modules/archive/ems_reporting.tf b/tf-modules/archive/ems_reporting.tf index eb1c9af3efaa..931907943f49 100644 --- a/tf-modules/archive/ems_reporting.tf +++ b/tf-modules/archive/ems_reporting.tf @@ -6,7 +6,7 @@ resource "aws_lambda_function" "ems_distribution_report" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/emsDistributionReport/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 900 memory_size = 320 environment { @@ -60,7 +60,7 @@ resource "aws_lambda_function" "ems_product_metadata_report" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/emsProductMetadataReport/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 320 environment { @@ -121,7 +121,7 @@ resource "aws_lambda_function" "ems_ingest_report" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/emsIngestReport/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 320 dead_letter_config { diff --git a/tf-modules/archive/execute_migrations.tf b/tf-modules/archive/execute_migrations.tf index 85afccff6030..3d75ba6ffc4f 100644 --- a/tf-modules/archive/execute_migrations.tf +++ b/tf-modules/archive/execute_migrations.tf @@ -130,7 +130,7 @@ resource "aws_lambda_function" "execute_migrations" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/executeMigrations/lambda.zip") handler = "index.handler" role = aws_iam_role.migration_processing.arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 1024 environment { diff --git a/tf-modules/archive/index_from_database.tf b/tf-modules/archive/index_from_database.tf index 7bae6a1265cc..ec9dc313eb7a 100644 --- a/tf-modules/archive/index_from_database.tf +++ b/tf-modules/archive/index_from_database.tf @@ -4,7 +4,7 @@ resource "aws_lambda_function" "index_from_database" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/indexFromDatabase/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 512 environment { diff --git a/tf-modules/archive/log2elasticsearch.tf b/tf-modules/archive/log2elasticsearch.tf index d36d449c41a5..d966a4412c14 100644 --- a/tf-modules/archive/log2elasticsearch.tf +++ b/tf-modules/archive/log2elasticsearch.tf @@ -11,7 +11,7 @@ resource "aws_lambda_function" "log2elasticsearch" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/indexer/lambda.zip") handler = "index.logHandler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 320 dead_letter_config { diff --git a/tf-modules/archive/reconciliation_report.tf b/tf-modules/archive/reconciliation_report.tf index ddf69ea83c4c..64e6b4d8d067 100644 --- a/tf-modules/archive/reconciliation_report.tf +++ b/tf-modules/archive/reconciliation_report.tf @@ -6,7 +6,7 @@ resource "aws_lambda_function" "create_reconciliation_report" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/createReconciliationReport/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 256 environment { diff --git a/tf-modules/archive/sns2elasticsearch.tf b/tf-modules/archive/sns2elasticsearch.tf index 8b68ddcac2a2..ef17aa684c29 100644 --- a/tf-modules/archive/sns2elasticsearch.tf +++ b/tf-modules/archive/sns2elasticsearch.tf @@ -11,7 +11,7 @@ resource "aws_lambda_function" "sns2elasticsearch" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/indexer/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 320 dead_letter_config { diff --git a/tf-modules/distribution/main.tf b/tf-modules/distribution/main.tf index 9339a0d56377..2ba9f6b75e38 100644 --- a/tf-modules/distribution/main.tf +++ b/tf-modules/distribution/main.tf @@ -128,7 +128,7 @@ resource "aws_lambda_function" "s3_credentials" { source_code_hash = filebase64sha256("${path.module}/dist/src.zip") handler = "index.handler" role = aws_iam_role.s3_credentials_lambda.arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 10 memory_size = 320 vpc_config { diff --git a/tf-modules/ingest/lambda-functions.tf b/tf-modules/ingest/lambda-functions.tf index 8d6b037744d5..6840cb0eacba 100644 --- a/tf-modules/ingest/lambda-functions.tf +++ b/tf-modules/ingest/lambda-functions.tf @@ -4,7 +4,7 @@ resource "aws_lambda_function" "fallback_consumer" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/messageConsumer/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 256 dead_letter_config { @@ -33,7 +33,7 @@ resource "aws_lambda_function" "kinesis_inbound_event_logger" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/payloadLogger/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 128 environment { @@ -55,7 +55,7 @@ resource "aws_lambda_function" "kinesis_outbound_event_logger" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/payloadLogger/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 512 environment { @@ -77,7 +77,7 @@ resource "aws_lambda_function" "message_consumer" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/messageConsumer/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 256 environment { @@ -105,7 +105,7 @@ resource "aws_lambda_function" "schedule_sf" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/sfScheduler/lambda.zip") handler = "index.schedule" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 192 dead_letter_config { @@ -132,7 +132,7 @@ resource "aws_lambda_function" "sf2snsEnd" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/sfSnsBroadcast/lambda.zip") handler = "index.end" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 128 environment { @@ -154,7 +154,7 @@ resource "aws_lambda_function" "sf2snsStart" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/sfSnsBroadcast/lambda.zip") handler = "index.start" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 128 environment { @@ -176,7 +176,7 @@ resource "aws_lambda_function" "sf_semaphore_down" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/sfSemaphoreDown/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 100 memory_size = 512 environment { @@ -199,7 +199,7 @@ resource "aws_lambda_function" "sf_sns_report" { source_code_hash = filebase64sha256("${path.module}/../../tasks/sf-sns-report/dist/lambda.zip") handler = "index.handler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 300 memory_size = 1024 environment { @@ -221,7 +221,7 @@ resource "aws_lambda_function" "sqs2sf" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/sfStarter/lambda.zip") handler = "index.sqs2sfHandler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 200 memory_size = 128 environment { @@ -243,7 +243,7 @@ resource "aws_lambda_function" "sqs2sfThrottle" { source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/sfStarter/lambda.zip") handler = "index.sqs2sfThrottleHandler" role = var.lambda_processing_role_arn - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 200 memory_size = 128 environment { diff --git a/tf-modules/report-executions/main.tf b/tf-modules/report-executions/main.tf index 99f2c4c16149..32c9c609491b 100644 --- a/tf-modules/report-executions/main.tf +++ b/tf-modules/report-executions/main.tf @@ -13,7 +13,7 @@ resource "aws_lambda_function" "report_executions" { function_name = "${var.prefix}-reportExecutions" role = "${aws_iam_role.report_executions_lambda_role.arn}" handler = "index.handler" - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 30 memory_size = 128 diff --git a/tf-modules/report-granules/main.tf b/tf-modules/report-granules/main.tf index 1914283f5cd4..7330409f6f43 100644 --- a/tf-modules/report-granules/main.tf +++ b/tf-modules/report-granules/main.tf @@ -13,7 +13,7 @@ resource "aws_lambda_function" "report_granules" { function_name = "${var.prefix}-reportGranules" role = "${aws_iam_role.report_granules_lambda_role.arn}" handler = "index.handler" - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 30 memory_size = 256 diff --git a/tf-modules/report-pdrs/main.tf b/tf-modules/report-pdrs/main.tf index 4253d7bf1300..5118c5999d25 100644 --- a/tf-modules/report-pdrs/main.tf +++ b/tf-modules/report-pdrs/main.tf @@ -9,7 +9,7 @@ resource "aws_lambda_function" "report_pdrs" { function_name = "${var.prefix}-reportPdrs" role = "${aws_iam_role.report_pdrs_lambda_role.arn}" handler = "index.handler" - runtime = "nodejs8.10" + runtime = "nodejs10.x" timeout = 30 memory_size = 128 From 812202baf98ce0cda17810c423625f4ca154fa8d Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Mon, 16 Dec 2019 22:53:42 -0700 Subject: [PATCH 5/9] Update CHANGELOG --- CHANGELOG.md | 5 +++++ tasks/discover-granules/package.json | 2 +- tasks/discover-pdrs/package.json | 2 +- tasks/files-to-granules/package.json | 2 +- tasks/hello-world/package.json | 2 +- tasks/move-granules/package.json | 2 +- tasks/parse-pdr/package.json | 2 +- tasks/pdr-status-check/package.json | 2 +- tasks/post-to-cmr/package.json | 2 +- tasks/queue-granules/package.json | 2 +- tasks/queue-pdrs/package.json | 2 +- tasks/sf-sns-report/package.json | 2 +- tasks/sync-granule/package.json | 2 +- tasks/test-processing/package.json | 2 +- 14 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdfadf7eb6f0..2d3d05b204e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Updated + +- **CUMULUS-1626** + - Updates Cumulus to use node10/CMA 1.1.2 for all of its internal lambdas in prep for AWS node 8 EOL + ## [v1.14.4] - 2019-10-28 ### Fixed diff --git a/tasks/discover-granules/package.json b/tasks/discover-granules/package.json index 121e6110eea7..de3d0f781cce 100644 --- a/tasks/discover-granules/package.json +++ b/tasks/discover-granules/package.json @@ -39,7 +39,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "@cumulus/test-data": "1.14.4", "lodash.get": "^4.4.2" diff --git a/tasks/discover-pdrs/package.json b/tasks/discover-pdrs/package.json index 44f4f410e0dd..3a6468100f73 100644 --- a/tasks/discover-pdrs/package.json +++ b/tasks/discover-pdrs/package.json @@ -38,7 +38,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "lodash.get": "^4.4.2" }, diff --git a/tasks/files-to-granules/package.json b/tasks/files-to-granules/package.json index 41e27b1857ae..0cdf811e47e0 100644 --- a/tasks/files-to-granules/package.json +++ b/tasks/files-to-granules/package.json @@ -32,7 +32,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "lodash.flatten": "^4.4.0", "lodash.get": "^4.4.2", "lodash.keyby": "^4.6.0" diff --git a/tasks/hello-world/package.json b/tasks/hello-world/package.json index 7765137d14d3..5c9728935f8d 100644 --- a/tasks/hello-world/package.json +++ b/tasks/hello-world/package.json @@ -36,7 +36,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8" + "@cumulus/cumulus-message-adapter-js": "^1.1.0" }, "devDependencies": { "ava": "^2.1.0", diff --git a/tasks/move-granules/package.json b/tasks/move-granules/package.json index f3be0050dc10..33b220600f41 100644 --- a/tasks/move-granules/package.json +++ b/tasks/move-granules/package.json @@ -43,7 +43,7 @@ "dependencies": { "@cumulus/cmrjs": "1.14.4", "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "lodash.clonedeep": "^4.5.0", "lodash.flatten": "^4.4.0", diff --git a/tasks/parse-pdr/package.json b/tasks/parse-pdr/package.json index 99116843f535..f8bc262079cd 100644 --- a/tasks/parse-pdr/package.json +++ b/tasks/parse-pdr/package.json @@ -32,7 +32,7 @@ }, "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "@cumulus/test-data": "1.14.4", "lodash.clonedeep": "^4.5.0", diff --git a/tasks/pdr-status-check/package.json b/tasks/pdr-status-check/package.json index 96005aa335df..1a3bd7273b2b 100644 --- a/tasks/pdr-status-check/package.json +++ b/tasks/pdr-status-check/package.json @@ -36,7 +36,7 @@ }, "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "lodash.get": "^4.4.2" }, diff --git a/tasks/post-to-cmr/package.json b/tasks/post-to-cmr/package.json index c3c52ba807a1..7c8899676a53 100644 --- a/tasks/post-to-cmr/package.json +++ b/tasks/post-to-cmr/package.json @@ -38,7 +38,7 @@ "dependencies": { "@cumulus/cmrjs": "1.14.4", "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "lodash.keyby": "^4.6.0" }, "devDependencies": { diff --git a/tasks/queue-granules/package.json b/tasks/queue-granules/package.json index 5cf854caf73a..2d64ffe171d7 100644 --- a/tasks/queue-granules/package.json +++ b/tasks/queue-granules/package.json @@ -33,7 +33,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "lodash.get": "^4.4.2" }, diff --git a/tasks/queue-pdrs/package.json b/tasks/queue-pdrs/package.json index ca1d436baafc..8c1d02cfc39f 100644 --- a/tasks/queue-pdrs/package.json +++ b/tasks/queue-pdrs/package.json @@ -33,7 +33,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "lodash.get": "^4.4.2" }, diff --git a/tasks/sf-sns-report/package.json b/tasks/sf-sns-report/package.json index 14a3fb64e83e..52a8a2ff5245 100644 --- a/tasks/sf-sns-report/package.json +++ b/tasks/sf-sns-report/package.json @@ -35,7 +35,7 @@ "dependencies": { "@cumulus/api": "1.14.4", "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "lodash.get": "^4.4.2", "lodash.isobject": "^3.0.2" diff --git a/tasks/sync-granule/package.json b/tasks/sync-granule/package.json index 27bca27ad8c4..21fdb1aa005b 100644 --- a/tasks/sync-granule/package.json +++ b/tasks/sync-granule/package.json @@ -36,7 +36,7 @@ }, "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/ingest": "1.14.4", "p-map": "^2.1.0" }, diff --git a/tasks/test-processing/package.json b/tasks/test-processing/package.json index 2ece25d38b93..bf54e437cb11 100644 --- a/tasks/test-processing/package.json +++ b/tasks/test-processing/package.json @@ -20,7 +20,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "1.14.4", - "@cumulus/cumulus-message-adapter-js": "^1.0.8", + "@cumulus/cumulus-message-adapter-js": "^1.1.0", "@cumulus/integration-tests": "1.13.1", "lodash.clonedeep": "^4.5.0" }, From 115109efd1c39002f1ee8a40b9bf60d112f97df3 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Mon, 16 Dec 2019 22:57:16 -0700 Subject: [PATCH 6/9] Update CMA layer --- example/app/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/app/config.yml b/example/app/config.yml index 4e4a7fa7cbac..9ffb1b5901e7 100644 --- a/example/app/config.yml +++ b/example/app/config.yml @@ -61,7 +61,7 @@ default: # test configuration ends cmaDir: '/opt/' - sandbox_cma_layer: arn:aws:lambda:us-east-1:596205514787:layer:Cumulus_Message_Adapter_test:16 + sandbox_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:10 sit_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:2 cma_layer: '{{sandbox_cma_layer}}' prefix: 'test-cumulus-integration' From 6a4aea9994b51974291783748cf56d9be10aa96f Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Wed, 18 Dec 2019 11:11:44 -0700 Subject: [PATCH 7/9] Fix cherry-pick/pulled lint errors in es-index test --- packages/api/tests/serial/es/test-es-indexer.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/api/tests/serial/es/test-es-indexer.js b/packages/api/tests/serial/es/test-es-indexer.js index fa85bbc82623..10b75d07124c 100644 --- a/packages/api/tests/serial/es/test-es-indexer.js +++ b/packages/api/tests/serial/es/test-es-indexer.js @@ -588,16 +588,16 @@ test.serial('parsePayload correctly parses AWS Linux 2 style console output', as test.serial('parsePayload correctly handles unparseable record', async (t) => { const parsePayload = indexer.__get__('parsePayload'); - const payload = { + const testPayload = { message: 'INFO MESSAGE', - sender: 'AWS sender', + sender: 'AWS sender', executions: 'some execution value', timestamp: '2018-06-01T17:45:27.108Z', version: '1' }; const expected = { message: 'INFO MESSAGE', - sender: 'AWS sender', + sender: 'AWS sender', executions: 'some execution value', timestamp: '2018-06-01T17:45:27.108Z', version: '1', @@ -605,6 +605,6 @@ test.serial('parsePayload correctly handles unparseable record', async (t) => { pid: 1, name: 'cumulus' }; - const actual = parsePayload(payload); + const actual = parsePayload(testPayload); t.deepEqual(actual, expected); }); From 4095166fc99c6637f34f9b415a831173cece6034 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Wed, 18 Dec 2019 11:37:00 -0700 Subject: [PATCH 8/9] Update cumulus-ecs-task/async-operation to use node 10 version of latest images --- docs/data-cookbooks/run-tasks-in-lambda-or-docker.md | 2 +- docs/workflows/developing-workflow-tasks.md | 2 +- example/app/config.yml | 2 +- packages/deployment/app/config.yml | 2 +- packages/deployment/test/fixtures/config.json | 6 +++--- tf-modules/archive/async_operation.tf | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/data-cookbooks/run-tasks-in-lambda-or-docker.md b/docs/data-cookbooks/run-tasks-in-lambda-or-docker.md index 5f2ceb93d187..150586086d19 100644 --- a/docs/data-cookbooks/run-tasks-in-lambda-or-docker.md +++ b/docs/data-cookbooks/run-tasks-in-lambda-or-docker.md @@ -64,7 +64,7 @@ Given it has been discovered this task can no longer run in AWS Lambda, it can b services: QueueGranules: docker: true - image: cumuluss/cumulus-ecs-task:1.2.5 + image: cumuluss/cumulus-ecs-task:1.4.0 memory: 4000 count: 1 envs: diff --git a/docs/workflows/developing-workflow-tasks.md b/docs/workflows/developing-workflow-tasks.md index ca0f0571e782..86319b42276b 100644 --- a/docs/workflows/developing-workflow-tasks.md +++ b/docs/workflows/developing-workflow-tasks.md @@ -32,7 +32,7 @@ ECS activities require a docker image. The docker image is defined as part of th registry: dockerhub services: EcsTaskHelloWorld: - image: cumuluss/cumulus-ecs-task:1.2.3 + image: cumuluss/cumulus-ecs-task:1.4.0 cpu: 800 memory: 1500 count: 1 diff --git a/example/app/config.yml b/example/app/config.yml index 9ffb1b5901e7..ba822445e597 100644 --- a/example/app/config.yml +++ b/example/app/config.yml @@ -103,7 +103,7 @@ default: username: cumulususer services: EcsTaskHelloWorld: - image: cumuluss/cumulus-ecs-task:1.3.0 + image: cumuluss/cumulus-ecs-task:1.4.0 cpu: 400 memory: 700 minTasks: 1 diff --git a/packages/deployment/app/config.yml b/packages/deployment/app/config.yml index 357c865cb003..c9eebca320ee 100644 --- a/packages/deployment/app/config.yml +++ b/packages/deployment/app/config.yml @@ -143,7 +143,7 @@ default: tasks: AsyncOperation: - image: cumuluss/async-operation:26 + image: cumuluss/async-operation:27 cpu: 400 memory: 700 count: 1 diff --git a/packages/deployment/test/fixtures/config.json b/packages/deployment/test/fixtures/config.json index 0aa6cdc21d84..4a0ab08c06a8 100644 --- a/packages/deployment/test/fixtures/config.json +++ b/packages/deployment/test/fixtures/config.json @@ -49,7 +49,7 @@ "desiredInstances": 2, "tasks": { "AsyncOperation": { - "image": "cumuluss/async-operation:25", + "image": "cumuluss/async-operation:27", "cpu": 400, "memory": 700, "count": 1, @@ -69,7 +69,7 @@ "publicIp": true, "services": { "EcsTaskHelloWorld": { - "image": "cumuluss/cumulus-ecs-task:1.2.3", + "image": "cumuluss/cumulus-ecs-task:1.4.0", "cpu": 400, "memory": 700, "count": 1, @@ -104,7 +104,7 @@ } }, "EcsTaskHelloWorldSecond": { - "image": "cumuluss/cumulus-ecs-task:1.2.3", + "image": "cumuluss/cumulus-ecs-task:1.4.0", "cpu": 400, "memory": 700, "count": 2, diff --git a/tf-modules/archive/async_operation.tf b/tf-modules/archive/async_operation.tf index 9fb4330dc0e6..8890160b7176 100644 --- a/tf-modules/archive/async_operation.tf +++ b/tf-modules/archive/async_operation.tf @@ -18,7 +18,7 @@ resource "aws_ecs_task_definition" "async_operation" { "value": "${data.aws_region.current.name}" } ], - "image": "cumuluss/async-operation:26", + "image": "cumuluss/async-operation:27", "memoryReservation": 700, "logConfiguration": { "logDriver": "awslogs", From bd57cb0bf0c0a42c123d02a28bb9c9c7fb224483 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Thu, 19 Dec 2019 13:10:37 -0700 Subject: [PATCH 9/9] Update packages/api/es/indexer.js Co-Authored-By: Mark Boyd --- packages/api/es/indexer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/es/indexer.js b/packages/api/es/indexer.js index 96d955f11aaf..2ad6094ed8a8 100644 --- a/packages/api/es/indexer.js +++ b/packages/api/es/indexer.js @@ -41,7 +41,7 @@ async function createIndex(esClient, indexName) { /** - * Parses a StepFunction log payload and returns a es logsrecord object + * Parses a StepFunction log payload and returns an Elasticsearch log record object * * @param {Object} payload - Stepfunction log payload * @returns {Object} - ElasticSearch log record