From db3d4f9502e3f992d56ead66d6e4f519402755c8 Mon Sep 17 00:00:00 2001 From: Chris Barrett Date: Sun, 19 Sep 2021 18:10:17 -0500 Subject: [PATCH] Support for building pygeoapi as a lambda container and deploying to AWS Lambda (#2) (#709) --- aws-lambda/README.md | 48 ++++++- aws-lambda/container/.serverless-wsgi | 1 + aws-lambda/container/Dockerfile | 122 ++++++++++++++++++ aws-lambda/container/lambda-entry.sh | 6 + aws-lambda/container/serverless.yml | 107 +++++++++++++++ aws-lambda/container/wsgi.py | 9 ++ .../pygeoapi-config.yml} | 0 aws-lambda/{ => function}/serverless.yml | 0 8 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 aws-lambda/container/.serverless-wsgi create mode 100644 aws-lambda/container/Dockerfile create mode 100644 aws-lambda/container/lambda-entry.sh create mode 100644 aws-lambda/container/serverless.yml create mode 100644 aws-lambda/container/wsgi.py rename aws-lambda/{pygeoapi-serverless-config.yml => function/pygeoapi-config.yml} (100%) rename aws-lambda/{ => function}/serverless.yml (100%) diff --git a/aws-lambda/README.md b/aws-lambda/README.md index 678b27a..3b2e86d 100644 --- a/aws-lambda/README.md +++ b/aws-lambda/README.md @@ -46,7 +46,7 @@ Move serverless configs to root directory: ```bash mv serverless.yml .. -mv pygeoapi-server-config.yml .. +mv pygeoapi-config.yml .. cd .. ``` @@ -83,3 +83,49 @@ serverless deploy --function app ``` When deployed, the output will show the URL the app has been deployed to. + +## node/serverless lambda container + +In the case where your pygeoapi instance is too large to deploy as a lambda function (250MB) you can build and deploy +a docker image of pygeoapi with the lamda runtime interface installed. + +Move serverless configs to root directory: + +```bash +mv container/serverless.yml ../.. +mv container/DockerFile ../.. +``` + +*note the files below come from the serverless-wsgi node plugin, and ideally this should be part of a build process +```bash +cd container/ +npm install serverless +serverless plugin install -n serverless-wsgi +mv node_modules/serverless-wsgi/serverless-wsgi.py ../.. +mv node_modules/serverless-wsgi/wsgi_handler.py ../.. +mv container/wsgi.py ../.. +mv container/.serverless-wsgi ../.. +rm -rf container/node_modules +cd ../.. +``` + +# to build docker container +```bash +docker build -t pygeo-lambda-container . +``` + +Once built, you need to deploy to ECR. This can also be accomplished with a change to the serverless configuration. +Depending on environment permissions, you may need to create a ECR repo with appropriate policies first. + +```bash +AWS_PROFILE= aws ecr get-login-password --region | docker login --username AWS --password-stdin +docker tag pygeo-lambda-container:latest :latest +docker push :latest +``` + + +Deploy stack using serverless. + +``` +AWS_PROFILE= sls deploy -s +``` \ No newline at end of file diff --git a/aws-lambda/container/.serverless-wsgi b/aws-lambda/container/.serverless-wsgi new file mode 100644 index 0000000..1770c6d --- /dev/null +++ b/aws-lambda/container/.serverless-wsgi @@ -0,0 +1 @@ +{"app":"pygeoapi/flask_app.APP","text_mime_types":["application/ld+json"]} \ No newline at end of file diff --git a/aws-lambda/container/Dockerfile b/aws-lambda/container/Dockerfile new file mode 100644 index 0000000..a358f09 --- /dev/null +++ b/aws-lambda/container/Dockerfile @@ -0,0 +1,122 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Just van den Broecke +# Francesco Bartoli +# Angelos Tzotsos +# +# Copyright (c) 2020 Tom Kralidis +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2020 Francesco Bartoli +# Copyright (c) 2021 Angelos Tzotsos +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +FROM ubuntu:focal + +LABEL maintainer="Just van den Broecke " + +# Docker file for full geoapi server with libs/packages for all providers. +# Server runs with gunicorn. You can override ENV settings. +# Defaults: +# SCRIPT_NAME=/ +# CONTAINER_NAME=pygeoapi +# CONTAINER_HOST=0.0.0.0 +# CONTAINER_PORT=80 +# WSGI_WORKERS=4 +# WSGI_WORKER_TIMEOUT=6000 +# WSGI_WORKER_CLASS=gevent + +# Calls entrypoint.sh to run. Inspect it for options. +# Contains some test data. Also allows you to verify by running all unit tests. +# Simply run: docker run -it geopython/pygeoapi test +# Override the default config file /pygeoapi/local.config.yml +# via Docker Volume mapping or within a docker-compose.yml file. See example at +# https://github.com/geopython/demo.pygeoapi.io/tree/master/services/pygeoapi + +# Build arguments +# add "--build-arg BUILD_DEV_IMAGE=true" to Docker build command when building with test/doc tools + +# ARGS +ARG TIMEZONE="Europe/London" +ARG LOCALE="en_US.UTF-8" +ARG BUILD_DEV_IMAGE="false" +ARG ADD_DEB_PACKAGES="python3-gdal python3-psycopg2 python3-xarray python3-scipy python3-netcdf4 python3-rasterio python3-fiona python3-pandas python3-pyproj python3-elasticsearch python3-pymongo python3-zarr python3-dask python3-tinydb" +ARG FUNCTION_DIR="/pygeoapi" + + + + +# ENV settings +ENV TZ=${TIMEZONE} \ + DEBIAN_FRONTEND="noninteractive" \ + DEB_BUILD_DEPS="software-properties-common curl unzip" \ + DEB_PACKAGES="python3-pip python3-setuptools python3-distutils python3-yaml python3-dateutil python3-tz lsof python3-flask python3-flask-cors python3-unicodecsv python3-click python3-greenlet python3-gevent python3-wheel gunicorn libsqlite3-mod-spatialite ${ADD_DEB_PACKAGES}" \ + PYGEOAPI_CONFIG="local.config.yml" \ + PYGEOAPI_OPENAPI="pygoapi-test-openapi.yml" + + + +RUN mkdir -p /pygeoapi/pygeoapi +WORKDIR /pygeoapi +# Add files required for pip/setuptools +ADD requirements*.txt setup.py README.md /pygeoapi/ +ADD pygeoapi/__init__.py /pygeoapi/pygeoapi/ + +# Run all installs +RUN \ + # Install dependencies + apt-get update -y \ + && apt-get install -y --fix-missing --no-install-recommends ${DEB_BUILD_DEPS} \ + && add-apt-repository ppa:ubuntugis/ubuntugis-unstable \ + && apt-get --no-install-recommends install -y ${DEB_PACKAGES} \ + && echo "For ${TZ} date=$(date)" && echo "Locale=$(locale)" \ + # Install pygeoapi + && cd /pygeoapi \ + # Optionally add development/test/doc packages + && if [ "$BUILD_DEV_IMAGE" = "true" ] ; then pip3 install -r requirements-dev.txt; fi \ + && pip3 install -e . \ + # OGC schemas local setup + && mkdir /schemas.opengis.net \ + && curl -O http://schemas.opengis.net/SCHEMAS_OPENGIS_NET.zip \ + && unzip ./SCHEMAS_OPENGIS_NET.zip "ogcapi/*" -d /schemas.opengis.net \ + && rm -f ./SCHEMAS_OPENGIS_NET.zip \ + # Cleanup TODO: remove unused Locales and TZs + && apt-get remove --purge -y ${DEB_BUILD_DEPS} \ + && apt autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --target "/pygeoapi" awslambdaric +ADD . /pygeoapi + +COPY ./docker/default.config.yml /pygeoapi/local.config.yml +COPY ./docker/entry.sh /pygeoapi/entry.sh + + +ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/bin/aws-lambda-rie +#COPY entry.sh / +RUN chmod 755 /usr/bin/aws-lambda-rie /pygeoapi/entry.sh +ENTRYPOINT [ "/pygeoapi/entry.sh" ] +CMD [ "wsgi_handler.handler" ] +#ENTRYPOINT ["/pygeoapi/entrypoint.sh"] diff --git a/aws-lambda/container/lambda-entry.sh b/aws-lambda/container/lambda-entry.sh new file mode 100644 index 0000000..ae74843 --- /dev/null +++ b/aws-lambda/container/lambda-entry.sh @@ -0,0 +1,6 @@ +#!/bin/sh +if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then + exec /usr/bin/aws-lambda-rie /usr/bin/python3 -m awslambdaric $1 +else + exec /usr/bin/python3 -m awslambdaric $1 +fi diff --git a/aws-lambda/container/serverless.yml b/aws-lambda/container/serverless.yml new file mode 100644 index 0000000..866ba31 --- /dev/null +++ b/aws-lambda/container/serverless.yml @@ -0,0 +1,107 @@ +# ================================================================= +# +# Authors: David Bitner > +# +# Copyright (c) 2019 David Bitner +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +service: pygeoapi + +provider: + name: aws + region: us-west-2 + # rolePermissionsBoundary: < perm boundary arn here > + # deploymentBucket: < deployment bucket name here > + # role: < lambda execution role here > + timeout: 30 + + ecr: + images: + pygeo-lambda-container: + uri: < url to container image in ECR > + +functions: + app: + role: + Fn::GetAtt: + - pygeoapiIamRole + - Arn + image: + name: pygeo-lambda-container + events: + - http: ANY / + - http: 'ANY {proxy+}' + entrypoint: + - '/entry.sh' + +resources: + Resources: + pygeoapiIamRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - states.amazonaws.com + - events.amazonaws.com + - lambda.amazonaws.com + - ec2.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: pygeo-role + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "ec2:DescribeNetworkInterfaces" + - "ec2:CreateNetworkInterface" + - "ecr:GetDownloadUrlForLayer" + - "ecr:PutImage" + - "ecr:InitiateLayerUpload" + - "ecr:UploadLayerPart" + - "ecr:CompleteLayerUpload" + - "ecr:DescribeRepositories" + - "ecr:GetRepositoryPolicy" + - "ecr:ListImages" + - "ecr:GetAuthorizationToken" + - "ecr:BatchCheckLayerAvailability" + - "ecr:BatchGetImage" + Resource: "*" + RoleName: ${self:service}-${self:provider.stage}-role + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaRole + - arn:aws:iam::aws:policy/CloudWatchFullAccess + PermissionsBoundary: + Fn::Sub: + - "< perm boundary arn >" + - accountId: + Ref: "AWS::AccountId" + +plugins: + - serverless-wsgi diff --git a/aws-lambda/container/wsgi.py b/aws-lambda/container/wsgi.py new file mode 100644 index 0000000..5c3e748 --- /dev/null +++ b/aws-lambda/container/wsgi.py @@ -0,0 +1,9 @@ +import os + +os.environ['PYGEOAPI_CONFIG'] = 'pygeoapi-test-config.yml' +os.environ['PYGEOAPI_OPENAPI'] = 'pygeoapi-test-openapi.yml' + +from pygeoapi.flask_app import APP + +if __name__ == "__main__": + APP.run() diff --git a/aws-lambda/pygeoapi-serverless-config.yml b/aws-lambda/function/pygeoapi-config.yml similarity index 100% rename from aws-lambda/pygeoapi-serverless-config.yml rename to aws-lambda/function/pygeoapi-config.yml diff --git a/aws-lambda/serverless.yml b/aws-lambda/function/serverless.yml similarity index 100% rename from aws-lambda/serverless.yml rename to aws-lambda/function/serverless.yml