SageMaker Serverless Inference

123456789101112
import boto3
import sagemaker
from sagemaker.serverless import ServerlessInferenceConfig
from sagemaker.pytorch import PyTorchModel

# Initialize clients
sm_client = boto3.client('sagemaker')
session = sagemaker.Session()
role = 'arn:aws:iam::123456789012:role/SageMakerExecutionRole'

# Define serverless configuration
serverless_config = ServerlessInferenceConfig(

123456789101112
# Multi-stage build for minimal image size
FROM python:3.10-slim as builder

WORKDIR /opt/ml

# Install build dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
    build-essential \
    && rm -rf /var/lib/apt/lists/*

# Create virtual environment and install dependencies
COPY requirements.txt .

123456789101112
import boto3
import json
from datetime import datetime

sagemaker = boto3.client('sagemaker')

# Configuration
model_name = f"sentiment-model-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
endpoint_config_name = f"{model_name}-config"
endpoint_name = "sentiment-serverless-endpoint"

# Step 1: Create the Model

123456789101112
import os
import json
import torch
import logging
from transformers import AutoTokenizer, AutoModelForSequenceClassification

logger = logging.getLogger(__name__)

# Global variables for model caching
model = None
tokenizer = None
device = None

123456789101112
import boto3
import sagemaker
from sagemaker.model import Model
from sagemaker.serverless import ServerlessInferenceConfig
import json
import time

# Initialize clients
sm_client = boto3.client('sagemaker')
session = sagemaker.Session()
role = sagemaker.get_execution_role()

123456789101112
import json
import torch
import logging
import uuid
from transformers import AutoModelForSequenceClassification, AutoTokenizer

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def model_fn(model_dir):
    """Load model and tokenizer from model directory."""
    logger.info(f'Loading model from {model_dir}')

123456789101112
import boto3
import json
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
import time

sm_runtime = boto3.client('sagemaker-runtime')

ENDPOINTS = {
    'classifier': os.environ['CLASSIFIER_ENDPOINT'],
    'ner': os.environ['NER_ENDPOINT'],
    'sentiment': os.environ['SENTIMENT_ENDPOINT']

123456789101112
{
  "widgets": [
    {
      "type": "metric",
      "properties": {
        "title": "Invocation Count",
        "metrics": [
          ["AWS/SageMaker", "Invocations", "EndpointName", "${EndpointName}", "VariantName", "AllTraffic", {"stat": "Sum", "period": 60}]
        ],
        "view": "timeSeries",
        "region": "${AWS::Region}"
      }