Cumulus 1626 node 10

.nvmrc

@@ -1 +1 @@
-8.10
+10.16.3

CHANGELOG.md

@@ -37,6 +37,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 
+- **CUMULUS-1626**
+  - Updates Cumulus to use node10/CMA 1.1.2 for all of it's internal lambdas in prep for AWS node 8 EOL
+
 - **CUMULUS-1578**
   - Updates SAML launchpad configuration to authorize via configured userGroup.
    [See the NASA specific documentation (protected)](https://wiki.earthdata.nasa.gov/display/CUMULUS/Cumulus+SAML+Launchpad+Integration)

bamboo/bootstrap-tf-deployment.sh

@@ -69,7 +69,7 @@ echo "Deploying Cumulus example to $DEPLOYMENT"
   -input=false \
   -var-file="../deployments/sandbox.tfvars" \
   -var-file="../deployments/$DEPLOYMENT.tfvars" \
-  -var "cumulus_message_adapter_lambda_layer_arn=arn:aws:lambda:us-east-1:$AWS_ACCOUNT_ID:layer:Cumulus_Message_Adapter:9" \
+  -var "cumulus_message_adapter_lambda_layer_arn=arn:aws:lambda:us-east-1:$AWS_ACCOUNT_ID:layer:Cumulus_Message_Adapter:10" \
   -var "cmr_username=$CMR_USERNAME" \
   -var "cmr_password=$CMR_PASSWORD" \
   -var "cmr_client_id=cumulus-core-$DEPLOYMENT" \

bamboo/docker-compose.yml

@@ -37,7 +37,7 @@ services:
     environment:
       SERVICES: "kinesis,lambda,s3,sns,sqs,dynamodb,cloudwatch,cloudwatchlogs"
   build_env:
-    image: jlkovarik/cumulus_build_env:1
+    image: jlkovarik/cumulus_build_env:2
     volumes:
       - ../:/source/cumulus
     environment:

docs/data-cookbooks/sns.md

@@ -113,7 +113,7 @@ resource "aws_lambda_function" "test_lambda" {
   source_code_hash = filebase64sha256("./testLambda.zip")
   handler          = "index.handler"
   role             = module.cumulus.lambda_processing_role_arn
-  runtime          = "nodejs8.10"
+  runtime          = "nodejs10.x"
 }
 
 resource "aws_sns_topic_subscription" "test_lambda" {

docs/workflows/developing-workflow-tasks.md

@@ -28,7 +28,7 @@ For a step-by-step example of using the `cumulus_ecs_module`, please see the rel
 
 ### Cumulus Docker Image
 
-ECS activities require a docker image.  Cumulus provides a docker image ([source](https://github.com/nasa/cumulus-ecs-task) for node 8.10 lambdas on dockerhub: [cumuluss/cumulus-ecs-task](https://hub.docker.com/r/cumuluss/cumulus-ecs-task).
+ECS activities require a docker image.  Cumulus provides a docker image ([source](https://github.com/nasa/cumulus-ecs-task) for node 10x+ lambdas on dockerhub: [cumuluss/cumulus-ecs-task](https://hub.docker.com/r/cumuluss/cumulus-ecs-task).
 
 ### Alternate Docker Images
 

docs/workflows/lambda.md

@@ -12,7 +12,7 @@ AWS provides great getting started guide for building Lambdas in the [developer
 
 Cumulus currently supports the following environments for Cumulus Message Adapter enabled functions:
 
-* [Node.js 8.10](https://docs.aws.amazon.com/lambda/latest/dg/programming-model.html)
+* [Node.js 10.16.3](https://docs.aws.amazon.com/lambda/latest/dg/programming-model.html)
 * [Java 8](https://docs.aws.amazon.com/lambda/latest/dg/java-programming-model.html)
 * [Python 2.7, 3.6](https://docs.aws.amazon.com/lambda/latest/dg/python-programming-model.html)
 
@@ -31,7 +31,7 @@ resource "aws_lambda_function" "myfunction" {
   source_code_hash = filebase64sha256("/path/to/zip/lambda.zip")
   handler          = "index.handler"
   role             = module.cumulus.lambda_processing_role_arn
-  runtime          = "nodejs8.10"
+  runtime          = "nodejs10.x"
 
   tags = { Deployment = var.prefix }
 

example/cumulus-tf/lambdas.tf

@@ -4,7 +4,7 @@ resource "aws_lambda_function" "async_operation_fail" {
   source_code_hash = filebase64sha256("${path.module}/../lambdas/asyncOperations/lambda.zip")
   handler          = "index.fail"
   role             = module.cumulus.lambda_processing_role_arn
-  runtime          = "nodejs8.10"
+  runtime          = "nodejs10.x"
 
   tags = local.default_tags
 
@@ -20,7 +20,7 @@ resource "aws_lambda_function" "async_operation_success" {
   source_code_hash = filebase64sha256("${path.module}/../lambdas/asyncOperations/lambda.zip")
   handler          = "index.success"
   role             = module.cumulus.lambda_processing_role_arn
-  runtime          = "nodejs8.10"
+  runtime          = "nodejs10.x"
 
   tags = local.default_tags
 
@@ -36,7 +36,7 @@ resource "aws_lambda_function" "sns_s3_test" {
   source_code_hash = filebase64sha256("${path.module}/../lambdas/snsS3Test/lambda.zip")
   handler          = "index.handler"
   role             = module.cumulus.lambda_processing_role_arn
-  runtime          = "nodejs8.10"
+  runtime          = "nodejs10.x"
 
   environment {
     variables = {

example/cumulus-tf/modules/s3_access_test/main.tf

@@ -17,7 +17,7 @@ resource "aws_lambda_function" "s3_acccess_test" {
   source_code_hash = filebase64sha256("${path.module}/../../../lambdas/s3AccessTest/lambda.zip")
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
-  runtime          = "nodejs8.10"
+  runtime          = "nodejs10.x"
 
   tags = local.default_tags
 }

example/package.json

@@ -56,7 +56,8 @@
     "@cumulus/sf-sns-report": "1.16.0",
     "@cumulus/sync-granule": "1.16.0",
     "@cumulus/test-processing": "1.16.0",
-    "aws-sdk": "^2.227.1"
+    "aws-sdk": "^2.585.0",
+    "ssh2-streams": "^0.4.8"
   },
   "devDependencies": {
     "@cumulus/test-data": "1.16.0",

package.json

@@ -37,7 +37,7 @@
     "url": "https://github.com/nasa/cumulus"
   },
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "bin": {
     "build-tasks-doc": "./bin/build-tasks-doc.js"
@@ -76,9 +76,10 @@
     "simple-git": "^1.96.0"
   },
   "dependencies": {
-    "aws-sdk": "^2.238.1",
+    "aws-sdk": "^2.585.0",
     "fs-extra": "^5.0.0",
     "latest-version": "^4.0.0",
-    "semver": "^5.5.0"
+    "semver": "^5.5.0",
+    "ssh2-streams": "^0.4.8"
   }
 }

packages/api/.babelrc

@@ -4,7 +4,7 @@
   "presets": [
     ["env", {
       "targets": {
-        "node": "8.10"
+        "node": "10.16.3"
       }
     }]
   ],

packages/api/ecs/async-operation/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:8.10
+FROM node:10.16.3
 
 USER root
 RUN sed -i -e '/jessie-updates/d' /etc/apt/sources.list

packages/api/ecs/async-operation/package.json

@@ -1,10 +1,10 @@
 {
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "dependencies": {
     "@cumulus/logger": "^1.13.0",
-    "aws-sdk": "^2.279.1",
+    "aws-sdk": "^2.585.0",
     "got": "^9.2.2",
     "lodash.iserror": "^3.1.1",
     "p-retry": "^2.0.0"

packages/api/es/indexer.js

@@ -38,6 +38,50 @@ async function createIndex(esClient, indexName) {
   log.info(`Created esIndex ${indexName}`);
 }
 
+
+/**
+ * Parses a StepFunction log payload  and returns a es logsrecord object
+ *
+ * @param {Object} payload - Stepfunction log payload
+ * @returns {Object} - ElasticSearch log record
+ */
+function parsePayload(payload) {
+  let record;
+  try {
+    // cumulus log message has extra aws messages before the json message,
+    // only the json message should be logged to elasticsearch.
+    // example message:
+    // 2018-06-0 1T17:45:27.108Z a714a0ef-f141-4e52-9661-58ca2233959a
+    // {"level": "info", "timestamp": "2018-06-01T17:45:27.108Z",
+    // "message": "uploaded s3://bucket/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf.met"}
+    const entryParts = payload.message.trim().split('\t');
+    // cumulus log message
+    const messageStartIndex = entryParts.findIndex((e) => e.startsWith('{'));
+    if (entryParts.length >= 3 && messageStartIndex
+    && entryParts[entryParts.length - 1].endsWith('}')) {
+      record = JSON.parse(entryParts.slice(messageStartIndex).join('\t'));
+      record.RequestId = entryParts[1];
+    } else { // other logs e.g. cumulus-ecs-task
+      record = JSON.parse(payload.message);
+    }
+    // level is number in elasticsearch
+    if (isString(record.level)) record.level = log.convertLogLevel(record.level);
+  } catch (e) {
+    record = {
+      message: payload.message.trim(),
+      sender: payload.sender,
+      executions: payload.executions,
+      timestamp: payload.timestamp,
+      version: payload.version,
+      level: 30,
+      pid: 1,
+      name: 'cumulus'
+    };
+  }
+  return record;
+}
+
+
 /**
  * Extracts info from a stepFunction message and indexes it to
  * an ElasticSearch
@@ -51,40 +95,10 @@ async function createIndex(esClient, indexName) {
 async function indexLog(esClient, payloads, index = defaultIndexAlias, type = 'logs') {
   const body = [];
 
-  payloads.forEach((p) => {
-    body.push({ index: { _index: index, _type: type, _id: p.id } });
-    let record;
-    try {
-      // cumulus log message has extra aws messages before the json message,
-      // only the json message should be logged to elasticsearch.
-      // example message:
-      // 2018-06-01T17:45:27.108Z a714a0ef-f141-4e52-9661-58ca2233959a
-      // {"level": "info", "timestamp": "2018-06-01T17:45:27.108Z",
-      // "message": "uploaded s3://bucket/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf.met"}
-      const entryParts = p.message.trim().split('\t');
-      // cumulus log message
-      if (entryParts.length >= 3 && entryParts[2].startsWith('{')
-      && entryParts[entryParts.length - 1].endsWith('}')) {
-        record = JSON.parse(entryParts.slice(2).join('\t'));
-        record.RequestId = entryParts[1];
-      } else { // other logs e.g. cumulus-ecs-task
-        record = JSON.parse(p.message);
-      }
-      // level is number in elasticsearch
-      if (isString(record.level)) record.level = log.convertLogLevel(record.level);
-    } catch (e) {
-      record = {
-        message: p.message.trim(),
-        sender: p.sender,
-        executions: p.executions,
-        timestamp: p.timestamp,
-        version: p.version,
-        level: 30,
-        pid: 1,
-        name: 'cumulus'
-      };
-    }
-    body.push(record);
+  payloads.forEach((payload) => {
+    body.push({ index: { _index: index, _type: type, _id: payload.id } });
+    const parsedPayload = parsePayload(payload);
+    body.push(parsedPayload);
   });
 
   const actualEsClient = esClient || (await Search.es());
@@ -175,6 +189,7 @@ function indexProvider(esClient, payload, index = defaultIndexAlias, type = 'pro
  * @param  {string} type     - Elasticsearch type (default: rule)
  * @returns {Promise} Elasticsearch response
  */
+
 function indexRule(esClient, payload, index = defaultIndexAlias, type = 'rule') {
   return genericRecordUpdate(esClient, payload.name, payload, index, type);
 }

packages/api/package.json

@@ -4,7 +4,7 @@
   "description": "Lambda functions for handling all daac's API operations",
   "main": "index.js",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "bin": {
     "cumulus-api": "./bin/cli.js"
@@ -58,7 +58,7 @@
     "@mapbox/dyno": "^1.4.2",
     "ajv": "^5.2.2",
     "aws-elasticsearch-connector": "^8.2.0",
-    "aws-sdk": "^2.238.1",
+    "aws-sdk": "^2.585.0",
     "aws-serverless-express": "^3.3.5",
     "body-parser": "^1.18.3",
     "boolean": "^0.1.3",

packages/api/tests/serial/es/test-es-indexer.js

@@ -2,6 +2,7 @@
 
 const test = require('ava');
 const sinon = require('sinon');
+const rewire = require('rewire');
 const fs = require('fs');
 const path = require('path');
 const aws = require('@cumulus/common/aws');
@@ -11,7 +12,7 @@ const { constructCollectionId } = require('@cumulus/common/collection-config-sto
 const StepFunctions = require('@cumulus/common/StepFunctions');
 const workflows = require('@cumulus/common/workflows');
 
-const indexer = require('../../../es/indexer');
+const indexer = rewire('../../../es/indexer');
 const { Search } = require('../../../es/search');
 const models = require('../../../models');
 const { fakeGranuleFactory, fakeCollectionFactory, deleteAliases } = require('../../../lib/testUtils');
@@ -492,3 +493,51 @@ test.serial('Create new index - index already exists', async (t) => {
 
   await esClient.indices.delete({ index: newIndex });
 });
+
+test.serial('parsePayload correctly parses AWS Linux style console output', async (t) => {
+  const parsePayload = indexer.__get__('parsePayload');
+  const expected = {
+    some: 'key',
+    sender: 'some sender',
+    message: 'a messaage',
+    RequestId: 'a714a0ef-f141-4e52-9661-58ca2233959a'
+  };
+  const actual = parsePayload({ sender: 'fixture_sender', message: '2018-06-01T17:45:27.108Z\ta714a0ef-f141-4e52-9661-58ca2233959a\t{"some": "key", "sender": "some sender", "message": "a messaage"}' });
+  t.deepEqual(actual, expected);
+});
+
+test.serial('parsePayload correctly parses AWS Linux 2 style console output', async (t) => {
+  const parsePayload = indexer.__get__('parsePayload');
+  const expected = {
+    some: 'key',
+    sender: 'some sender',
+    message: 'a messaage',
+    RequestId: 'a714a0ef-f141-4e52-9661-58ca2233959a'
+  };
+  const actual = parsePayload({ sender: 'fixture_sender', message: '2018-06-01T17:45:27.108Z\ta714a0ef-f141-4e52-9661-58ca2233959a\tINFO\t{"some": "key", "sender": "some sender", "message": "a messaage"}' });
+  t.deepEqual(actual, expected);
+});
+
+
+test.serial('parsePayload correctly handles unparseable record', async (t) => {
+  const parsePayload = indexer.__get__('parsePayload');
+  const testPayload = {
+    message: 'INFO MESSAGE',
+    sender: 'AWS sender',
+    executions: 'some execution value',
+    timestamp: '2018-06-01T17:45:27.108Z',
+    version: '1'
+  };
+  const expected = {
+    message: 'INFO MESSAGE',
+    sender: 'AWS sender',
+    executions: 'some execution value',
+    timestamp: '2018-06-01T17:45:27.108Z',
+    version: '1',
+    level: 30,
+    pid: 1,
+    name: 'cumulus'
+  };
+  const actual = parsePayload(testPayload);
+  t.deepEqual(actual, expected);
+});

packages/checksum/package.json

@@ -3,7 +3,7 @@
   "version": "1.16.0",
   "description": "Cumulus checksum utilities",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "test": "ava",

packages/cmr-client/package.json

@@ -2,7 +2,7 @@
   "name": "@cumulus/cmr-client",
   "version": "1.16.0",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "build-docs": "jsdoc2md --heading-depth 2 --template templates/API.hbs CMR.js CMRSearchConceptQueue.js > API.md",

packages/cmrjs/package.json

@@ -3,7 +3,7 @@
   "version": "1.16.0",
   "description": "A node SDK for CMR",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=10.16.3"
   },
   "scripts": {
     "test": "ava",