arclogs

#!/usr/bin/perl -wT
#
#  arclogs
#
#  Retrieves log file entries for a job from Elasticsearch.  Usage:
#
#  arclogs --cluster=flux --days=7 job_id
#
#  If not specified, days defaults to 7.  Legal values for days ranges from 0
#  to 59.
#

use strict;
use warnings;


BEGIN {
    unshift( @INC, '/sw/lsa/centos7/arc-admin-utils/1.0/perl5/share/perl5' );
    unshift( @INC, '/sw/lsa/centos7/arc-admin-utils/1.0/perl5/lib64/perl5' );
    %ENV = (); # for security
}

use Search::Elasticsearch;
use Getopt::Long;
use Data::Dumper;

$0 =~ m/([^\/]+)$/;
my $program_name = $1 || 'arclogs';

my $opt_cluster = 'flux';
my $opt_days    = 7;
my $opt_help    = 0;

if ( open( C, "</var/spool/torque/server_name" ) ) {
    $opt_cluster = <C>;
    chomp($opt_cluster);
    $opt_cluster = 'flux' if $opt_cluster eq 'nyx.arc-ts.umich.edu';
    $opt_cluster = 'armis' if $opt_cluster eq 'armis.arc-ts.umich.edu';
    close(C);
}

my $result = GetOptions(
  'cluster=s' => \$opt_cluster,
  'days=i'    => \$opt_days,
  'help'      => \$opt_help,
  );
if ( ! $result || $opt_help || scalar( @ARGV ) != 1 ) {
    print STDERR "\nusage: \n";
    print STDERR "  ${program_name} [options] JOB-NUMBER\n\n";
    print STDERR "  options:\n";
    print STDERR "    -h, --help   show this help message and exit\n";
    print STDERR "    --days=N     search the most recent N days of logs (default: 7)\n";
    print STDERR "    --cluster=(flux|armis)\n";
    print STDERR "                 which cluster's logs to search (default: the cluster that\n";
    print STDERR "                 ${program_name} is running on)\n\n";
    exit( $opt_help ? 0 : 1 );
}

if ( $opt_days < 0 || $opt_days > 59 ) {
    print STDERR "${program_name}: days must be in the range 0..59\n";
    exit( 1 );
}

my $cluster_rm = 'nyx';
if ( $opt_cluster eq 'armis' ) {
  $cluster_rm = 'armis';
} elsif ( $opt_cluster ne 'flux' ) {
    print STDERR "${program_name}: unknown cluster name\n";
    exit( 1 );
}

my $job_id = $ARGV[0];
$job_id =~ s/\.${cluster_rm}\.arc-ts.umich\.edu\s*$//;
if ( $job_id !~ /^\d+(\[\d*\])?$/ ) {
    print STDERR "$0: bad job id\n";
    exit( 1 );
}
$job_id = quotemeta( $job_id );

my $d = $opt_days + 1;
my $t = time();
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime( $t );
my $indices = sprintf( 'logstash-hpc-%s-joblogs-v3-%04d.%02d.%02d', $opt_cluster,
    $year + 1900, $mon + 1, $mday );
$d--;
while ( $d > 0 ) {
    $t -= 86400;
    ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime( $t );
    $indices = sprintf( 'logstash-hpc-%s-joblogs-v3-%04d.%02d.%02d,',
        $opt_cluster, $year + 1900, $mon + 1, $mday ) . $indices;
    $d--;
}

my $e = Search::Elasticsearch->new( nodes => 'es.arc-ts.umich.edu:9200' );

my $results = $e->search(
    index => $indices,
    body  => {
        'size' => 10000,
        'sort' => [ { '@timestamp' => { 'order' => 'asc' } } ],
        query => {
            'query_string' => {
                # Match any of the following:
                #   14467816.nyx.arc-ts.umich.edu
                #   job 14467816
                #   Job '14467816'
                #   job:14467816
                # Note that the specific results we are relying on depend on
                # the elasticsearch analyser chosen for the index.
                'query' => "\"${job_id}.${cluster_rm}.arc-ts.umich.edu\" OR \"job ${job_id}\""
            }
        }
    }
);
#print Dumper($results) . "\n";

my $logs = $results->{'hits'}->{'hits'};
for my $log (@$logs) {
    if ( defined( $log->{'_source'} ) ) {
        my $entry = $log->{'_source'};
        my $timestamp = $entry->{'@timestamp'}; # e.g., 2014-12-15T08:00:55.362-05:00
        $timestamp = substr( $timestamp, 0, 23 ); # get rid of TZ offset
        substr( $timestamp, 10, 1 ) = ' '; # Replace T with space
        my $type = defined( $entry->{'type'} ) ? $entry->{'type'} : 'unknown';
        if ( $type eq 'syslog' ) {
            $type = $entry->{'program'} if defined( $entry->{'program'} );
        } else {
            $type =~ s/log$//;
        }
        if ( defined( $entry->{'event'} ) ) {
            my $message;
            if ( $type eq 'pbsacct' ) {
                # 'event' does not contain all the info, use 'message' instead
                $message = $entry->{'message'};
                # remove the date
                $message =~ s/^\d{2}\/\d{2}\/\d{4}\s+\d{2}:\d{2}:\d{2};//;
            }
            else {
                # normally we use 'event' because it has less noise than
                # 'message'
                $message = $entry->{'event'};
            }
            print "$timestamp $type $message\n";
        } else {
            print 'NO MESSAGE ' . Dumper( $entry ) . "\n";
        }
    } else {
            print 'NO ENTRY: ' . Dumper( $log ) . "\n";
    }
}
#print Dumper($logs) . "\n";