Backport node10

.nvmrc

@@ -1 +1 @@
-8.10
+10.16.3

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Updated
+
+- **CUMULUS-1626**
+  - Updates Cumulus to use node10/CMA 1.1.2 for all of its internal lambdas in prep for AWS node 8 EOL
+
 ## [v1.14.4] - 2019-10-28
 
 ### Fixed

bamboo/docker-compose.yml

@@ -37,7 +37,7 @@ services:
     environment:
       SERVICES: 'kinesis,lambda,s3,sns,sqs,dynamodb,dynamodbstreams,cloudwatch,cloudwatchlogs'
   build_env:
-    image: jlkovarik/cumulus_build_env:1
+    image: jlkovarik/cumulus_build_env:2
     volumes:
       - ../:/source/cumulus
     environment:

docs/data-cookbooks/run-tasks-in-lambda-or-docker.md

@@ -64,7 +64,7 @@ Given it has been discovered this task can no longer run in AWS Lambda, it can b
     services:
       QueueGranules:
         docker: true
-        image: cumuluss/cumulus-ecs-task:1.2.5
+        image: cumuluss/cumulus-ecs-task:1.4.0
         memory: 4000
         count: 1
         envs:

docs/deployment/README.md

@@ -35,7 +35,7 @@ The process involves:
 ### Linux/MacOS software requirements
 
 * git
-* [node 8.10](https://nodejs.org/en/) (use [nvm](https://github.com/creationix/nvm) to upgrade/downgrade)
+* [node 10.16.3](https://nodejs.org/en/) (use [nvm](https://github.com/creationix/nvm) to upgrade/downgrade)
 * [npm](https://www.npmjs.com/get-npm)
 * sha1sum or md5sha1sum
 * zip

docs/workflows/developing-workflow-tasks.md

@@ -32,7 +32,7 @@ ECS activities require a docker image. The docker image is defined as part of th
       registry: dockerhub
     services:
       EcsTaskHelloWorld:
-        image: cumuluss/cumulus-ecs-task:1.2.3
+        image: cumuluss/cumulus-ecs-task:1.4.0
         cpu: 800
         memory: 1500
         count: 1

example/README.md

@@ -172,7 +172,7 @@ https://abc123.execute-api.us-east-1.amazonaws.com:7000/DEV/my-protected/path/to
 An S3 Access lambda is needed in the us-west-2 region to run the integration tests. To initially create the lambda, run:
 
 ```
-aws lambda create-function --region us-west-2  --function-name <STACK>-S3AccessTest --zip-file fileb://app/build/cloudformation/<ZIP>-S3AccessTest.zip  --role arn:aws:iam::<AWS_ACCOUNT_ID>:role/<PREFIX>-lambda-processing  --handler index.handler --runtime nodejs8.10 --profile <NGAP Profile>
+aws lambda create-function --region us-west-2  --function-name <STACK>-S3AccessTest --zip-file fileb://app/build/cloudformation/<ZIP>-S3AccessTest.zip  --role arn:aws:iam::<AWS_ACCOUNT_ID>:role/<PREFIX>-lambda-processing  --handler index.handler --runtime nodejs10.x --profile <NGAP Profile>
 ```
 
 Replace `<AWS_ACCOUNT_ID>` with your account Id, `<STACK>` with your stack name, `<PREFIX>` with your iam prefix name, and use your NGAP profile. An S3AccessTest zip file `<ZIP>` can be found under `app/build/` following a stack deployment (either in `cloudformation` or `workflow_lambda_versions` depending on your stack configuration). The version of the zip file that you upload does not matter, but you need to deploy something to us-west-2 so that Bamboo can update it with current lambdas.

example/app/config.yml

@@ -61,7 +61,7 @@ default:
   # test configuration ends
 
   cmaDir: '/opt/'
-  sandbox_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:6
+  sandbox_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:10
   sit_cma_layer: arn:aws:lambda:us-east-1:{{AWS_ACCOUNT_ID}}:layer:Cumulus_Message_Adapter:2
   cma_layer: '{{sandbox_cma_layer}}'
   prefix: 'test-cumulus-integration'
@@ -103,7 +103,7 @@ default:
       username: cumulususer
     services:
       EcsTaskHelloWorld:
-        image: cumuluss/cumulus-ecs-task:1.3.0
+        image: cumuluss/cumulus-ecs-task:1.4.0
         cpu: 400
         memory: 700
         minTasks: 1

example/package.json

@@ -5,7 +5,7 @@
   "private": true,
   "main": "index.js",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "test": "jasmine && npm run parallel-tests",
@@ -55,11 +55,12 @@
     "@cumulus/sf-sns-report": "1.14.4",
     "@cumulus/sync-granule": "1.14.4",
     "@cumulus/test-processing": "1.14.4",
-    "aws-sdk": "^2.227.1",
+    "aws-sdk": "^2.585.0",
     "child-process-promise": "^2.2.1",
     "lodash.differencewith": "^4.5.0",
     "lodash.isequal": "^4.5.0",
-    "p-retry": "^2.0.0"
+    "p-retry": "^2.0.0",
+    "ssh2-streams": "^0.4.8"
   },
   "devDependencies": {
     "@cumulus/test-data": "1.14.4",

package.json

@@ -36,7 +36,7 @@
     "url": "https://github.com/nasa/cumulus"
   },
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "bin": {
     "build-tasks-doc": "./bin/build-tasks-doc.js"
@@ -71,10 +71,11 @@
     "eslint-plugin-unicorn": "^4.0.3",
     "lerna": "^3.13.2",
     "nyc": "^14.0.0",
-    "simple-git": "^1.96.0"
+    "simple-git": "^1.96.0",
+    "ssh2-streams": "^0.4.8"
   },
   "dependencies": {
-    "aws-sdk": "^2.238.1",
+    "aws-sdk": "^2.585.0",
     "fs-extra": "^5.0.0",
     "latest-version": "^4.0.0",
     "semver": "^5.5.0"

packages/api/ecs/async-operation/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:8.10
+FROM node:10.16.3
 
 USER root
 RUN sed -i -e '/jessie-updates/d' /etc/apt/sources.list

packages/api/ecs/async-operation/package.json

@@ -1,10 +1,10 @@
 {
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "dependencies": {
     "@cumulus/logger": "^1.13.0",
-    "aws-sdk": "^2.279.1",
+    "aws-sdk": "^2.585.0",
     "got": "^9.2.2",
     "lodash.iserror": "^3.1.1",
     "p-retry": "^2.0.0"

packages/api/es/indexer.js

@@ -39,6 +39,50 @@ async function createIndex(esClient, indexName) {
   log.info(`Created esIndex ${indexName}`);
 }
 
+
+/**
+ * Parses a StepFunction log payload and returns an Elasticsearch log record object
+ *
+ * @param {Object} payload - Stepfunction log payload
+ * @returns {Object} - ElasticSearch log record
+ */
+function parsePayload(payload) {
+  let record;
+  try {
+    // cumulus log message has extra aws messages before the json message,
+    // only the json message should be logged to elasticsearch.
+    // example message:
+    // 2018-06-0 1T17:45:27.108Z a714a0ef-f141-4e52-9661-58ca2233959a
+    // {"level": "info", "timestamp": "2018-06-01T17:45:27.108Z",
+    // "message": "uploaded s3://bucket/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf.met"}
+    const entryParts = payload.message.trim().split('\t');
+    // cumulus log message
+    const messageStartIndex = entryParts.findIndex((e) => e.startsWith('{'));
+    if (entryParts.length >= 3 && messageStartIndex
+    && entryParts[entryParts.length - 1].endsWith('}')) {
+      record = JSON.parse(entryParts.slice(messageStartIndex).join('\t'));
+      record.RequestId = entryParts[1];
+    } else { // other logs e.g. cumulus-ecs-task
+      record = JSON.parse(payload.message);
+    }
+    // level is number in elasticsearch
+    if (isString(record.level)) record.level = log.convertLogLevel(record.level);
+  } catch (e) {
+    record = {
+      message: payload.message.trim(),
+      sender: payload.sender,
+      executions: payload.executions,
+      timestamp: payload.timestamp,
+      version: payload.version,
+      level: 30,
+      pid: 1,
+      name: 'cumulus'
+    };
+  }
+  return record;
+}
+
+
 /**
  * Extracts info from a stepFunction message and indexes it to
  * an ElasticSearch
@@ -52,40 +96,10 @@ async function createIndex(esClient, indexName) {
 async function indexLog(esClient, payloads, index = defaultIndexAlias, type = 'logs') {
   const body = [];
 
-  payloads.forEach((p) => {
-    body.push({ index: { _index: index, _type: type, _id: p.id } });
-    let record;
-    try {
-      // cumulus log message has extra aws messages before the json message,
-      // only the json message should be logged to elasticsearch.
-      // example message:
-      // 2018-06-01T17:45:27.108Z a714a0ef-f141-4e52-9661-58ca2233959a
-      // {"level": "info", "timestamp": "2018-06-01T17:45:27.108Z",
-      // "message": "uploaded s3://bucket/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf.met"}
-      const entryParts = p.message.trim().split('\t');
-      // cumulus log message
-      if (entryParts.length >= 3 && entryParts[2].startsWith('{')
-      && entryParts[entryParts.length - 1].endsWith('}')) {
-        record = JSON.parse(entryParts.slice(2).join('\t'));
-        record.RequestId = entryParts[1];
-      } else { // other logs e.g. cumulus-ecs-task
-        record = JSON.parse(p.message);
-      }
-      // level is number in elasticsearch
-      if (isString(record.level)) record.level = log.convertLogLevel(record.level);
-    } catch (e) {
-      record = {
-        message: p.message.trim(),
-        sender: p.sender,
-        executions: p.executions,
-        timestamp: p.timestamp,
-        version: p.version,
-        level: 30,
-        pid: 1,
-        name: 'cumulus'
-      };
-    }
-    body.push(record);
+  payloads.forEach((payload) => {
+    body.push({ index: { _index: index, _type: type, _id: payload.id } });
+    const parsedPayload = parsePayload(payload);
+    body.push(parsedPayload);
   });
 
   const actualEsClient = esClient || (await Search.es());
@@ -176,6 +190,7 @@ function indexProvider(esClient, payload, index = defaultIndexAlias, type = 'pro
  * @param  {string} type     - Elasticsearch type (default: rule)
  * @returns {Promise} Elasticsearch response
  */
+
 function indexRule(esClient, payload, index = defaultIndexAlias, type = 'rule') {
   return genericRecordUpdate(esClient, payload.name, payload, index, type);
 }

packages/api/package.json

@@ -4,7 +4,7 @@
   "description": "Lambda functions for handling all daac's API operations",
   "main": "index.js",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "bin": {
     "cumulus-api": "./bin/cli.js"
@@ -58,7 +58,7 @@
     "ajv": "^5.2.2",
     "archiver": "^2.1.1",
     "aws-elasticsearch-connector": "8.1.3",
-    "aws-sdk": "^2.238.1",
+    "aws-sdk": "^2.585.0",
     "aws-serverless-express": "^3.3.5",
     "basic-auth": "^1.1.0",
     "body-parser": "^1.18.3",

packages/api/tests/serial/es/test-es-indexer.js

@@ -2,6 +2,7 @@
 
 const test = require('ava');
 const sinon = require('sinon');
+const rewire = require('rewire');
 const fs = require('fs');
 const path = require('path');
 const aws = require('@cumulus/common/aws');
@@ -11,7 +12,7 @@ const { constructCollectionId } = require('@cumulus/common/collection-config-sto
 const { noop } = require('@cumulus/common/util');
 const StepFunctions = require('@cumulus/common/StepFunctions');
 
-const indexer = require('../../../es/indexer');
+const indexer = rewire('../../../es/indexer');
 const { Search } = require('../../../es/search');
 const models = require('../../../models');
 const { fakeGranuleFactory, fakeCollectionFactory, deleteAliases } = require('../../../lib/testUtils');
@@ -559,3 +560,51 @@ test.serial('Create new index - index already exists', async (t) => {
 
   await esClient.indices.delete({ index: newIndex });
 });
+
+test.serial('parsePayload correctly parses AWS Linux style console output', async (t) => {
+  const parsePayload = indexer.__get__('parsePayload');
+  const expected = {
+    some: 'key',
+    sender: 'some sender',
+    message: 'a messaage',
+    RequestId: 'a714a0ef-f141-4e52-9661-58ca2233959a'
+  };
+  const actual = parsePayload({ sender: 'fixture_sender', message: '2018-06-01T17:45:27.108Z\ta714a0ef-f141-4e52-9661-58ca2233959a\t{"some": "key", "sender": "some sender", "message": "a messaage"}' });
+  t.deepEqual(actual, expected);
+});
+
+test.serial('parsePayload correctly parses AWS Linux 2 style console output', async (t) => {
+  const parsePayload = indexer.__get__('parsePayload');
+  const expected = {
+    some: 'key',
+    sender: 'some sender',
+    message: 'a messaage',
+    RequestId: 'a714a0ef-f141-4e52-9661-58ca2233959a'
+  };
+  const actual = parsePayload({ sender: 'fixture_sender', message: '2018-06-01T17:45:27.108Z\ta714a0ef-f141-4e52-9661-58ca2233959a\tINFO\t{"some": "key", "sender": "some sender", "message": "a messaage"}' });
+  t.deepEqual(actual, expected);
+});
+
+
+test.serial('parsePayload correctly handles unparseable record', async (t) => {
+  const parsePayload = indexer.__get__('parsePayload');
+  const testPayload = {
+    message: 'INFO MESSAGE',
+    sender: 'AWS sender',
+    executions: 'some execution value',
+    timestamp: '2018-06-01T17:45:27.108Z',
+    version: '1'
+  };
+  const expected = {
+    message: 'INFO MESSAGE',
+    sender: 'AWS sender',
+    executions: 'some execution value',
+    timestamp: '2018-06-01T17:45:27.108Z',
+    version: '1',
+    level: 30,
+    pid: 1,
+    name: 'cumulus'
+  };
+  const actual = parsePayload(testPayload);
+  t.deepEqual(actual, expected);
+});

packages/checksum/package.json

@@ -3,7 +3,7 @@
   "version": "1.14.4",
   "description": "Cumulus checksum utilities",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "test": "ava",

packages/cmr-client/package.json

@@ -2,7 +2,7 @@
   "name": "@cumulus/cmr-client",
   "version": "1.14.4",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "build-docs": "jsdoc2md --heading-depth 3 --template templates/README.hbs CMR.js CMRSearchConceptQueue.js > README.md",

packages/cmrjs/package.json

@@ -3,7 +3,7 @@
   "version": "1.14.4",
   "description": "A node SDK for CMR",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "test": "ava",

packages/common/package.json

@@ -8,7 +8,7 @@
     "NASA"
   ],
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "publishConfig": {
     "access": "public"
@@ -47,7 +47,7 @@
     "@cumulus/logger": "1.14.4",
     "ajv": "^5.2.2",
     "async": "^2.0.0",
-    "aws-sdk": "^2.250.1",
+    "aws-sdk": "^2.585.0",
     "expect.js": "^0.3.1",
     "follow-redirects": "^1.2.4",
     "formidable": "^1.2.1",
@@ -82,7 +82,7 @@
     "p-retry": "^2.0.0",
     "pump": "^3.0.0",
     "randexp": "^0.4.9",
-    "ssh2": "^0.5.5",
+    "ssh2": "^0.8.7",
     "url-join": "^4.0.0",
     "uuid": "^3.2.1"
   },

packages/deployment/app/cloudformation.template.yml

@@ -519,7 +519,7 @@ Resources:
       Role: {{../iams.lambdaProcessingRoleArn}}
 {{/ifNotEquals}}
 {{/if}}
-      Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs8.10{{/if}}
+      Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs10.x{{/if}}
       Timeout: {{this.timeout}}
     {{# if this.deadletterqueue}}
       DeadLetterConfig: