Skip to content

Overview

Full script

The script below provides a good example of how to use the various JARVICE API endpoints to control and interact with a job in the Nimbix Public Cloud. The example is a bit superfluous, but that is with the intent of showing how to use as many of the JARVICE API endpoints as possible:

jarvice-job-exec

#!/bin/bash

#set -x  # Trace script execution

jarvice_api_url="https://cloud.nimbix.net/api"
jarvice_api_url+="/jarvice"

# Updating these values will modify the default job submit JSON below
jarvice_app="jarvice-filemanager"
jarvice_application_command="filemanager"
jarvice_application_geometry="1423x812"
jarvice_machine_type="n0"
jarvice_machine_nodes="1"
jarvice_vault_name="drop.jarvice.com"
jarvice_user_username=
jarvice_user_apikey=

# Parse JSON text to get values based upon given key
function get_json_value {
    json=$1
    key=$2
    echo "$json" | \
        python -c "import json,sys;obj=json.load(sys.stdin);print obj$key;" \
        2>/dev/null
}

function jarvice_api_endpoint {
    endpoint=$1
    jarvice_endpoint_url="$jarvice_api_url/$endpoint"

    # If $2 starts with '{', it must be JSON for POSTing
    if echo "$2" | grep -q '^ *{'; then
        curl -s -d "$2" "$jarvice_endpoint_url"
    else
        shift
        while [ $# -gt 0 ]; do
            echo $1
            extra_args+=" --data-urlencode $1"
            shift
        done

        # The job's number could be used instead of the job's name
        # Example: --data-urlencode "number=$jarvice_job_number"
        curl -s --get "$jarvice_endpoint_url" \
            --data-urlencode "username=$jarvice_user_username" \
            --data-urlencode "apikey=$jarvice_user_apikey" \
            --data-urlencode "name=$jarvice_job_name" \
            $extra_args
    fi
}

# Interaction function can be overridden with script from command line args.
# This default fucntion works with jarvice-filemanager to upload/download
# files to/from JARVICE vaults.
function jarvice_job_plugin {
    cmd="$1"
    src="$2"
    dst="$3"

    webdav_url="https://$jarvice_job_address/owncloud/remote.php/webdav/"
    curl="curl -u nimbix:$jarvice_job_password -s -k"
    curl+=" --retry-delay 1 --retry 30 --retry-connrefused"
    if [ "$cmd" = "--upload" ]; then
        echo "Uploading $src to $dst..."
        $curl --upload-file "$src" "$webdav_url$dst"
    elif [ "$cmd" = "--download" ]; then
        echo "Downloading $src to $dst..."
        $curl --output "$dst" "$webdav_url$src"
    else
        echo "jarvice_job_plugin function failed!  Unknown command '$cmd'!"
    fi
}

function jarvice_job_exec_usage {
    echo "Usage: $0 [options] -- <jarvice_job_plugin_options>"
    echo "Available [options]:"
    echo -e " --username\tJARVICE user username"
    echo -e " --apikey\tJARVICE user apikey"
    echo -e " --job-json\tJSON defining the JARVICE job to run"
    echo -e " --job-action\tExecute defined action for this app"
    echo -e " --job-output\tView job output upon successful completion"
    echo -e " --job-plugin\tPlugin file to source jarvice_job_plugin" \
        "function from"
    echo -e " --job-plugin-skip\tSkip execution of jarvice_job_plugin"
}

while [ $# -gt 0 ]; do
    case $1 in
        --help)
            jarvice_job_exec_usage
            exit 0
            ;;
        --username)
            jarvice_user_username=$2
            shift; shift
            ;;
        --apikey)
            jarvice_user_apikey=$2
            shift; shift
            ;;
        --job-json)
            jarvice_job_submit_json="$(cat $2)"
            shift; shift
            ;;
        --job-action)
            jarvice_job_action=$2
            shift; shift
            ;;
        --job-output)
            jarvice_job_output=y
            shift
            ;;
        --job-plugin)  # to override jarvice_job_plugin
            . $2
            shift; shift
            ;;
        --job-plugin-skip)
            jarvice_job_plugin_skip=y
            shift
            ;;
        --)  # args after "--" will be passed to jarvice_job_plugin
            shift
            break
            ;;
        *)
            jarvice_job_exec_usage
            exit 1
            ;;
    esac
done

# The JSON can be found under the "Preview Submission" tab when kicking off a
# job from https://cloud.nimbix.net/
[ -z "$jarvice_job_submit_json" ] && jarvice_job_submit_json=$(cat <<EOF
{
  "app": "$jarvice_app",
  "staging": false,
  "checkedout": false,
  "application": {
    "command": "$jarvice_application_command",
    "geometry": "$jarvice_application_geometry"
  },
  "machine": {
    "type": "$jarvice_machine_type",
    "nodes": $jarvice_machine_nodes
  },
  "vault": {
    "name": "$jarvice_vault_name",
    "force": false,
    "readonly": false
  },
  "user": {
    "username": "$jarvice_user_username",
    "apikey": "$jarvice_user_apikey"
  }
}
EOF
)

# If $jarvice_user_username or $jarvice_user_apikey were not specified on the
# command line, grab them from the job JSON.  They are required for
# interacting with the JARVICE API.
[ -z "$jarvice_user_username" ] && \
    jarvice_user_username=$(get_json_value "$jarvice_job_submit_json" \
        "['user']['username']")
[ -z "$jarvice_user_username" ] && echo "JARVICE username is empty!" \
    && jarvice_job_exec_usage && exit 1
[ -z "$jarvice_user_apikey" ] && \
    jarvice_user_apikey=$(get_json_value "$jarvice_job_submit_json" \
        "['user']['apikey']")
[ -z "$jarvice_user_apikey" ] && echo "JARVICE apikey is empty!" \
    && jarvice_job_exec_usage && exit 1

echo "Submitting JARVICE job..."
json_result=$(jarvice_api_endpoint "submit" "$jarvice_job_submit_json")
jarvice_job_name=$(get_json_value "$json_result" "['name']")
jarvice_job_number=$(get_json_value "$json_result" "['number']")

sleep 2
# Once we're able to get job info, we know everything is up and running.
echo "Getting JARVICE job info..."
while : ; do 
    sleep 1
    json_result=$(jarvice_api_endpoint "info")
    error=$(get_json_value "$json_result" "['error']")
    [ "$error" = "" ] && break
done
jarvice_job_address=$(get_json_value "$json_result" "['address']")
jarvice_job_actions=$(get_json_value "$json_result" "['actions']")
echo "JARVICE job '$jarvice_job_name' is running on host " \
    "'$jarvice_job_address'..."
echo "JARVICE job actions available:"
echo "$jarvice_job_actions"

echo "Getting connect info for JARVICE job '$jarvice_job_name'..."
json_result=$(jarvice_api_endpoint "connect")
jarvice_job_address=$(get_json_value "$json_result" "['address']")
jarvice_job_password=$(get_json_value "$json_result" "['password']")

# Applications with "actions" defined in their AppDef.json will return those
# actions when requesting job info.  They can be executed with the action EP.
if [ -n "$jarvice_job_action" ]; then
    echo "Executing action '$jarvice_job_action' for JARVICE job" \
        " '$jarvice_job_name'..."
    json_result=$(jarvice_api_endpoint "action" "action=$jarvice_job_action")
    echo "$json_result"
fi

# Run shell function which interacts with the running JARVICE job.
[ -z "$jarvice_job_plugin_skip" ] && jarvice_job_plugin "$@"

echo "Shutting down JARVICE job '$jarvice_job_name'..."
json_result=$(jarvice_api_endpoint "shutdown")

echo "Getting status for JARVICE job '$jarvice_job_name'..."
sleep 3  # Give the job some time to shutdown
json_result=$(jarvice_api_endpoint "status")
jarvice_job_status=$(get_json_value "$json_result" \
    "['$jarvice_job_number']['job_status']")

if [ "$jarvice_job_status" = "COMPLETED" ]; then
    echo "Successfully shut down JARVICE job '$jarvice_job_name'..."
    # Get all the output from the completed job
    if [ -n "$jarvice_job_output" ]; then
        echo "JARVICE job output:"
        jarvice_api_endpoint "output"
    fi
else
    echo "JARVICE job '$jarvice_job_name' may not be properly shutting down..."
    echo "Most recent JARVICE job status: $jarvice_job_status"
    echo "Tail of JARVICE job output: $jarvice_job_status"
    jarvice_api_endpoint "tail" "lines=100"
fi

# Still processing?  Forcefully terminate the job..
if [ "$jarvice_job_status" = "PROCESSING STARTING" ]; then
    echo "Terminating JARVICE job '$jarvice_job_name'..."
    json_result=$(jarvice_api_endpoint "terminate")
fi

The jarvice-job-exec script interacts with the jarvice-filemanager by default in order to upload and download files to and from JARVICE vaults. After downloading jarvice-job-exec with the title link, it can be executed like so:

./jarvice-job-exec --username <username> --apikey <apikey> -- --upload ./filename /data/filename
./jarvice-job-exec --username <username> --apikey <apikey> -- --download /data/filename ./filename

Executing jarvice-job-exec with the --help flag will print out it's usage:

Usage: ./jarvice-job-exec [options] -- <jarvice_job_plugin_options>
Available [options]:
 --username JARVICE user username
 --apikey   JARVICE user apikey
 --job-json JSON defining the JARVICE job to run
 --job-action   Execute defined action for this app
 --job-output   View job output upon successful completion
 --job-plugin   Plugin file to source jarvice_job_plugin function from
 --job-plugin-skip  Skip execution of jarvice_job_plugin

Custom job execution with jarvice-job-exec

As seen above, with the --job-json argument, it is possible to submit a JARVICE job with custom JSON using a local file. The JARVICE portal can be used to grab JSON which can be used as a starting point. When launching a job from the portal, click on the "Preview Submission" tab to copy and paste the job's JSON text.

Here is a JSON file one might use with --job-json to override the default jarvice-filemanager job started by jarvice-job-exec:

jarvice-job.json

{
  "app": "jarvice-filemanager",
  "staging": false,
  "checkedout": false,
  "application": {
    "command": "filemanager",
    "geometry": "1423x812"
  },
  "machine": {
    "type": "n0",
    "nodes": 1
  },
  "vault": {
    "name": "drop.jarvice.com",
    "force": false,
    "readonly": false
  },
  "user": {
    "username": "<username>",
    "apikey": "<apikey>"
  }
}

When specifying custom JSON with a username and apikey in it, it is no longer necessary to use the --username and --apikey arguments when executing jarvice-job-exec.

jarvice-filemanager-plugin.sh

When executing jarvice-job-exec with the --job-plugin flag, it is possible to provide customized code for interacting with the job. The following example overrides the jarvice_job_plugin function. Unlike the default jarvice-filemanager code in jarvice-job-exec, this custom plugin allows uploading and downloading of files in a single job run:

function jarvice_job_plugin_usage {
    echo "Available <jarvice_job_plugin_options>:"
    echo -e " --up <src> <dst>\tUpload local <src> file to remote <dst>"
    echo -e " --down <src> <dst>\tDownload remote <src> file to local <dst>"
}

function jarvice_job_plugin {
    while [ $# -gt 0 ]; do
        case $1 in
            --up)
                upload_src="$2"
                upload_dst="$3"
                shift; shift; shift
                ;;
            --down)
                download_src="$2"
                download_dst="$3"
                shift; shift; shift
                ;;
            *)
                jarvice_job_plugin_usage
                return
                ;;
        esac
    done

    webdav_url="https://$jarvice_job_address/owncloud/remote.php/webdav/"
    curl="curl -u nimbix:$jarvice_job_password -s -k"
    curl+=" --retry-delay 1 --retry 30 --retry-connrefused"
    if [ -n "$upload_src" -a -n "$upload_dst" ]; then
        echo "Uploading $upload_src to $upload_dst..."
        $curl --upload-file "$upload_src" "$webdav_url$upload_dst"
    fi
    if [ -n "$download_src" -a -n "$download_dst" ]; then
        echo "Downloading $download_src to $download_dst..."
        $curl --output "$download_dst" "$webdav_url$download_src"
    fi
}

With the above JSON and plugin code examples, jarvice-job-exec can be executed like so to customize JARVICE job execution and interaction:

./jarvice-job-exec --job-json ./jarvice-job.json --job-plugin ./jarvice-filemanager-plugin.sh -- --up ./filename /data/filename --down /data/filename ./filename

Step by step batch job example

In this section, we are going to submit an example job, interact with it, and close it. All endpoints details are given in the API page of this documentation. Note also that you need to adapt the api HTTP URL to your cluster.

First step is to submit a job. We are going to submit a very simple job, that will download a movie sample in H264, and convert it in H265 using FFMPEG. We will use a public docker hub ffmpeg image for that.

First, lets create our json file for the job (file will be written as ffmpeg_job.json file):

{
  "machine": {
    "type": "n1",
    "nodes": 1
  },
  "vault": {
    "name": "ephemeral",
    "readonly": false,
    "force": false
  },
  "user": {
    "username": "me",
    "apikey": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
  },
  "container": {
        "image": "jrottenberg/ffmpeg:latest",
        "jobscript": "/usr/local/bin/ffmpeg -stats -i https://test-videos.co.uk/vids/jellyfish/mp4/h264/1080/Jellyfish_1080_10s_2MB.mp4 -c:v libx265 -f mp4 $HOME/test.mp4"
  }
}

Remember to adapt machine, vault, and user dicts to your needs.

Then lets submit a batch job:

me@localhost:~$ curl -H 'Content-Type: application/json' -X POST -d @ffmpeg_job.json https://jarvice-api.cloud.nimbix.net/jarvice/batch
{
    "name": "20230906104200-995TB-jarvice-batch-me_s1",
    "number": 123065
me@localhost:~$

You can see we got as response a job name. We will use it in next steps.

Now lets request the status of our job:

me@localhost:~$ curl -X GET 'https://jarvice-api.cloud.nimbix.net/jarvice/status?username=me&apikey=XXXXXXXXXXXXXXXXXXXXXX&name=20230906104200-995TB-jarvice-batch-me_s1'
{
    "123065": {
        "job_name": "20230906104200-995TB-jarvice-batch-me_s1",
        "job_status": "PROCESSING STARTING",
        "job_substatus": 0,
        "job_start_time": 1693996925,
        "job_end_time": 0,
        "job_submit_time": 1693996920,
        "job_application": "jarvice-batch",
        "job_command": "Batch",
        "job_walltime": null,
        "job_project": null
    }
}me@localhost:~$ 

We can see that the job still havent started yet. If we wait some time and come back with the same command:

me@localhost:~$ curl -X GET 'https://jarvice-api.cloud.nimbix.net/jarvice/status?username=me&apikey=XXXXXXXXXXXXXXXXXXXXXX&name=20230906104200-995TB-jarvice-batch-me_s1'
{
    "123065": {
        "job_name": "20230906104200-995TB-jarvice-batch-me_s1",
        "job_status": "COMPLETED",
        "job_substatus": 0,
        "job_start_time": 1693996925,
        "job_end_time": 1693996942,
        "job_submit_time": 1693996920,
        "job_application": "jarvice-batch",
        "job_command": "Batch",
        "job_walltime": "00:00:17",
        "job_project": null
    }
me@localhost:~$

We can now see it is completed.

Lest request the logs output for this job now:

me@localhost:~$ curl -X GET 'https://jarvice-api.cloud.nimbix.net/jarvice/output?username=me&apikey=XXXXXXXXXXXXXXXXXXXXXX&name=20230906104200-995TB-jarvice-batch-me_s1'

INIT[1]: Configuring user: nimbix nimbix 505...
INIT[1]: Initializing networking...
INIT[1]: WARNING: Cross Memory Attach not available for MPI applications
INIT[1]: Platform fabric and MPI libraries successfully deployed
INIT[1]: Detected preferred MPI fabric provider: tcp
INIT[1]: Reading keys...
INIT[1]: Finalizing setup in application environment...
INIT[1]: Waiting for job configuration before executing application...
INIT[1]: hostname: jarvice-job-123065-x9m4v
INIT[1]: Injecting static ssh client.
INIT[1]: Starting SSHD server...
INIT[1]: Checking all nodes can be reached through ssh...
INIT[1]: SSH test success!
###############################################################################

ffmpeg version 4.1 Copyright (c) 2000-2018 the FFmpeg developers
  built with gcc 5.4.0 (Ubuntu 5.4.0-6ubuntu1~16.04.11) 20160609
  configuration: --disable-debug --disable-doc --disable-ffplay --enable-shared --enable-avresample --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-gpl --enable-libass --enable-libfreetype --enable-libvidstab --enable-libmp3lame --enable-libopenjpeg --enable-libopus --enable-libtheora --enable-libvorbis --enable-libvpx --enable-libx265 --enable-libxvid --enable-libx264 --enable-nonfree --enable-openssl --enable-libfdk_aac --enable-libkvazaar --enable-libaom --extra-libs=-lpthread --enable-postproc --enable-small --enable-version3 --extra-cflags=-I/opt/ffmpeg/include --extra-ldflags=-L/opt/ffmpeg/lib --extra-libs=-ldl --prefix=/opt/ffmpeg
  libavutil      56. 22.100 / 56. 22.100
  libavcodec     58. 35.100 / 58. 35.100
  libavformat    58. 20.100 / 58. 20.100
  libavdevice    58.  5.100 / 58.  5.100
  libavfilter     7. 40.101 /  7. 40.101
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  3.100 /  5.  3.100
  libswresample   3.  3.100 /  3.  3.100
  libpostproc    55.  3.100 / 55.  3.100
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'https://test-videos.co.uk/vids/jellyfish/mp4/h264/1080/Jellyfish_1080_10s_2MB.mp4':
  Metadata:
    major_brand     : isom
    minor_version   : 512
    compatible_brands: isomiso2avc1mp41
  Duration: 00:00:10.01, start: 0.000000, bitrate: 1676 kb/s
    Stream #0:0(und): Video: h264 (avc1 / 0x31637661), yuv420p, 1920x1080 [SAR 1:1 DAR 16:9], 1672 kb/s, 29.97 fps, 29.97 tbr, 11988 tbn, 59.94 tbc (default)
    Metadata:
      handler_name    : VideoHandler
Stream mapping:
  Stream #0:0 -> #0:0 (h264 (native) -> hevc (libx265))
Press [q] to stop, [?] for help
x265 [info]: HEVC encoder version 2.3
x265 [info]: build info [Linux][GCC 5.4.0][64 bit] 8bit+10bit+12bit
x265 [info]: using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX AVX2 FMA3 LZCNT BMI2
x265 [info]: Main profile, Level-4 (Main tier)
x265 [info]: Thread pool created using 16 threads
x265 [info]: Slices                              : 1
x265 [info]: frame threads / pool features       : 5 / wpp(17 rows)
x265 [info]: Coding QT: max CU size, min CU size : 64 / 8
x265 [info]: Residual QT: max TU size, max depth : 32 / 1 inter / 1 intra
x265 [info]: ME / range / subpel / merge         : hex / 57 / 2 / 2
x265 [info]: Keyframe min / max / scenecut / bias: 25 / 250 / 40 / 5.00
x265 [info]: Lookahead / bframes / badapt        : 20 / 4 / 2
x265 [info]: b-pyramid / weightp / weightb       : 1 / 1 / 0
x265 [info]: References / ref-limit  cu / depth  : 3 / on / on
x265 [info]: AQ: mode / str / qg-size / cu-tree  : 1 / 1.0 / 32 / 1
x265 [info]: Rate Control / qCompress            : CRF-28.0 / 0.60
x265 [info]: tools: rd=3 psy-rd=2.00 rskip signhide tmvp strong-intra-smoothing
x265 [info]: tools: lslices=6 deblock sao
Output #0, mp4, to '/home/nimbix/test.mp4':
  Metadata:
    major_brand     : isom
    minor_version   : 512
    compatible_brands: isomiso2avc1mp41
    encoder         : Lavf58.20.100
    Stream #0:0(und): Video: hevc (libx265) (hev1 / 0x31766568), yuv420p, 1920x1080 [SAR 1:1 DAR 16:9], q=2-31, 29.97 fps, 11988 tbn, 29.97 tbc (default)
    Metadata:
      handler_name    : VideoHandler
      encoder         : Lavc58.35.100 libx265
frame=  300 fps= 28 q=-0.0 Lsize=    2360kB time=00:00:09.90 bitrate=1950.6kbits/s speed=0.919x
video:2353kB audio:0kB subtitle:0kB other streams:0kB global headers:2kB muxing overhead: 0.270860%
x265 [info]: frame I:      2, Avg QP:24.36  kb/s: 7139.57
x265 [info]: frame P:     75, Avg QP:26.90  kb/s: 4460.35
x265 [info]: frame B:    223, Avg QP:34.48  kb/s: 1025.40
x265 [info]: Weighted P-Frames: Y:0.0% UV:0.0%
x265 [info]: consecutive B-frames: 2.6% 1.3% 1.3% 93.5% 1.3%

encoded 300 frames in 10.70s (28.05 fps), 1924.90 kb/s, Avg QP:32.52
me@localhost:~$

We can see our video was encoded in 10.70s. Since we used in this example an ephemeral vault, our output file is now lost. But you can use a persistent vault (check available ones in the portal) and store output file into /data, which is associated to requested vault during job execution.