ローカルLLMメッセンジャー:iPhoneでGenAIとチャット

git clone https://github.com/dockersamples/local-llm-messenger

.
├── LICENSE
├── README.md
├── app
│   ├── Dockerfile
│   ├── Pipfile
│   ├── Pipfile.lock
│   ├── default.ai
│   ├── log_conf.yaml
│   └── main.py
├── docker-compose.yaml
├── img
│   ├── banner.png
│   ├── lasers.gif
│   └── lollm-demo-1.gif
├── justfile
└── test
    ├── msg.json
    └── ollama.json

4 directories, 15 files

from dotenv import load_dotenv
import os, requests, time, openai, json, logging
from pprint import pprint
from typing import Union, List

from fastapi import FastAPI
from pydantic import BaseModel

from sendblue import Sendblue

SENDBLUE_API_KEY = os.environ.get("SENDBLUE_API_KEY")
SENDBLUE_API_SECRET = os.environ.get("SENDBLUE_API_SECRET")
openai.api_key = os.environ.get("OPENAI_API_KEY")
OLLAMA_API = os.environ.get("OLLAMA_API_ENDPOINT", "http://ollama:11434/api")
# could also use request.headers.get('referer') to do dynamically
CALLBACK_URL = os.environ.get("CALLBACK_URL")
MAX_WORDS = os.environ.get("MAX_WORDS")

def set_default_model(model: str):
    try:
        with open("default.ai", "w") as f:
            f.write(model)
            f.close()
            return
    except IOError:
        logger.error("Could not open file")
        exit(1)


def get_default_model() -> str:
    try:
        with open("default.ai") as f:
            default = f.readline().strip("\n")
            f.close()
            if default != "":
                return default
            else:
                set_default_model("llama2:latest")
                return ""
    except IOError:
        logger.error("Could not open file")
        exit(1)


def validate_model(model: str) -> bool:
    available_models = get_model_list()
    if model in available_models:
        return True
    else:
        return False


def get_ollama_model_list() -> List[str]:
    available_models = []
   
    tags = requests.get(OLLAMA_API + "/tags")
    all_models = json.loads(tags.text)
    for model in all_models["models"]:
        available_models.append(model["name"])
    return available_models


def get_openai_model_list() -> List[str]:
    return ["gpt-3.5-turbo", "dall-e-2"]


def get_model_list() -> List[str]:
    ollama_models = []
    openai_models = []
    all_models = []
    if "OPENAI_API_KEY" in os.environ:
        # print(openai.Model.list())
        openai_models = get_openai_model_list()

    ollama_models = get_ollama_model_list()
    all_models = ollama_models + openai_models
    return all_models


DEFAULT_MODEL = get_default_model()

if DEFAULT_MODEL == "":
    # This is probably the first run so we need to install a model
    if "OPENAI_API_KEY" in os.environ:
        print("No default model set. openai is enabled. using gpt-3.5-turbo")
        DEFAULT_MODEL = "gpt-3.5-turbo"
    else:
        print("No model found and openai not enabled. Installing llama2:latest")
        pull_data = '{"name": "llama2:latest","stream": false}'
        try:
            pull_resp = requests.post(OLLAMA_API + "/pull", data=pull_data)
            pull_resp.raise_for_status()
        except requests.exceptions.HTTPError as err:
            raise SystemExit(err)
        set_default_model("llama2:latest")
        DEFAULT_MODEL = "llama2:latest"


if validate_model(DEFAULT_MODEL):
    logger.info("Using model: " + DEFAULT_MODEL)
else:
    logger.error("Model " + DEFAULT_MODEL + " not available.")
    logger.info(get_model_list())

    pull_data = '{"name": "' + DEFAULT_MODEL + '","stream": false}'
    try:
        pull_resp = requests.post(OLLAMA_API + "/pull", data=pull_data)
        pull_resp.raise_for_status()
    except requests.exceptions.HTTPError as err:
        raise SystemExit(err)



def set_msg_send_style(received_msg: str):
    """Will return a style for the message to send based on matched words in received message"""
    celebration_match = ["happy"]
    shooting_star_match = ["star", "stars"]
    fireworks_match = ["celebrate", "firework"]
    lasers_match = ["cool", "lasers", "laser"]
    love_match = ["love"]
    confetti_match = ["yay"]
    balloons_match = ["party"]
    echo_match = ["what did you say"]
    invisible_match = ["quietly"]
    gentle_match = []
    loud_match = ["hear"]
    slam_match = []

    received_msg_lower = received_msg.lower()
    if any(x in received_msg_lower for x in celebration_match):
        return "celebration"
    elif any(x in received_msg_lower for x in shooting_star_match):
        return "shooting_star"
    elif any(x in received_msg_lower for x in fireworks_match):
        return "fireworks"
    elif any(x in received_msg_lower for x in lasers_match):
        return "lasers"
    elif any(x in received_msg_lower for x in love_match):
        return "love"
    elif any(x in received_msg_lower for x in confetti_match):
        return "confetti"
    elif any(x in received_msg_lower for x in balloons_match):
        return "balloons"
    elif any(x in received_msg_lower for x in echo_match):
        return "echo"
    elif any(x in received_msg_lower for x in invisible_match):
        return "invisible"
    elif any(x in received_msg_lower for x in gentle_match):
        return "gentle"
    elif any(x in received_msg_lower for x in loud_match):
        return "loud"
    elif any(x in received_msg_lower for x in slam_match):
        return "slam"
    else:
        return

class Msg(BaseModel):
    accountEmail: str
    content: str
    media_url: str
    is_outbound: bool
    status: str
    error_code: int | None = None
    error_message: str | None = None
    message_handle: str
    date_sent: str
    date_updated: str
    from_number: str
    number: str
    to_number: str
    was_downgraded: bool | None = None
    plan: str


class Callback(BaseModel):
    accountEmail: str
    content: str
    is_outbound: bool
    status: str
    error_code: int | None = None
    error_message: str | None = None
    message_handle: str
    date_sent: str
    date_updated: str
    from_number: str
    number: str
    to_number: str
    was_downgraded: bool | None = None
    plan: str



def msg_openai(msg: Msg, model="gpt-3.5-turbo"):
    """Sends a message to openai"""
    message_with_context = create_messages_from_context("openai")

    # Add the user's message and system context to the messages list
    messages = [
        {"role": "user", "content": msg.content},
        {"role": "system", "content": "You are an AI assistant. You will answer in haiku."},
    ]

    # Convert JSON strings to Python dictionaries and add them to messages
    messages.extend(
        [
            json.loads(line)  # Convert each JSON string back into a dictionary
            for line in message_with_context
        ]
    )

    # Send the messages to the OpenAI model
    gpt_resp = client.chat.completions.create(
        model=model,
        messages=messages,
    )

    # Append the system context to the context file
    append_context("system", gpt_resp.choices[0].message.content)

    # Send a message to the sender
    msg_response = sendblue.send_message(
        msg.from_number,
        {
            "content": gpt_resp.choices[0].message.content,
            "status_callback": CALLBACK_URL,
        },
    )
    
    return




def msg_ollama(msg: Msg, model=None):
    """Sends a message to the ollama endpoint"""
    if model is None:
        logger.error("Model is None when calling msg_ollama")
        return  # Optionally handle the case more gracefully

    ollama_headers = {"Content-Type": "application/json"}
    ollama_data = (
        '{"model":"' + model +
        '", "stream": false, "prompt":"' +
        msg.content +
        " in under " +
        str(MAX_WORDS) +  # Make sure MAX_WORDS is a string
        ' words"}'
    )
    ollama_resp = requests.post(
        OLLAMA_API + "/generate", headers=ollama_headers, data=ollama_data
    )
    response_dict = json.loads(ollama_resp.text)
    if ollama_resp.ok:
        send_style = set_msg_send_style(msg.content)
        append_context("system", response_dict["response"])
        msg_response = sendblue.send_message(
            msg.from_number,
            {
                "content": response_dict["response"],
                "status_callback": CALLBACK_URL,
                "send_style": send_style,
            },
        )
    else:
        msg_response = sendblue.send_message(
            msg.from_number,
            {
                "content": "I'm sorry, I had a problem processing that question. Please try again.",
                "status_callback": CALLBACK_URL,
            },
        )
    return

touch .env
SENDBLUE_API_KEY=your_sendblue_api_key
SENDBLUE_API_SECRET=your_sendblue_api_secret
OLLAMA_API_ENDPOINT=http://host.docker.internal:11434/api
OPENAI_API_KEY=your_openai_api_key

services:
  lollm:
    build: ./app
    # command:
      # - sleep
      # - 1d
    ports:
      - 8000:8000
    env_file: ./app/.env
    volumes:
      - ./run/lollm:/run/lollm
    depends_on:
      - ollama
    restart: unless-stopped
    network_mode: "host"
  ngrok:
    image: ngrok/ngrok:alpine
    command:
      - "http"
      - "8000"
      - "--log"
      - "stdout"
    environment:
      - NGROK_AUTHTOKEN=2i6iXXXXXXXXhpqk1aY1
    network_mode: "host"
  ollama:
    image: ollama/ollama
    ports:
      - 11434:11434
    volumes:
      - ./run/ollama:/home/ollama
    network_mode: "host"

$ docker compose up

[+] Running 4/4
 ✔ Container local-llm-messenger-ollama-1                           Create...                                          0.0s
 ✔ Container local-llm-messenger-ngrok-1                            Created                                            0.0s
 ✔ Container local-llm-messenger-lollm-1                            Recreat...                                         0.1s
 ! lollm Published ports are discarded when using host network mode                                                    0.0s
Attaching to lollm-1, ngrok-1, ollama-1
ollama-1  | 2024/06/20 03:14:46 routes.go:1011: INFO server config env="map[OLLAMA_DEBUG:false OLLAMA_FLASH_ATTENTION:false OLLAMA_HOST:http://0.0.0.0:11434 OLLAMA_KEEP_ALIVE: OLLAMA_LLM_LIBRARY: OLLAMA_MAX_LOADED_MODELS:1 OLLAMA_MAX_QUEUE:512 OLLAMA_MAX_VRAM:0 OLLAMA_MODELS:/root/.ollama/models OLLAMA_NOHISTORY:false OLLAMA_NOPRUNE:false OLLAMA_NUM_PARALLEL:1 OLLAMA_ORIGINS:[http://localhost https://localhost http://localhost:* https://localhost:* http://127.0.0.1 https://127.0.0.1 http://127.0.0.1:* https://127.0.0.1:* http://0.0.0.0 https://0.0.0.0 http://0.0.0.0:* https://0.0.0.0:* app://* file://* tauri://*] OLLAMA_RUNNERS_DIR: OLLAMA_TMPDIR:]"
ollama-1  | time=2024-06-20T03:14:46.308Z level=INFO source=images.go:725 msg="total blobs: 0"
ollama-1  | time=2024-06-20T03:14:46.309Z level=INFO source=images.go:732 msg="total unused blobs removed: 0"
ollama-1  | time=2024-06-20T03:14:46.309Z level=INFO source=routes.go:1057 msg="Listening on [::]:11434 (version 0.1.44)"
ollama-1  | time=2024-06-20T03:14:46.309Z level=INFO source=payload.go:30 msg="extracting embedded files" dir=/tmp/ollama2210839504/runners
ngrok-1   | t=2024-06-20T03:14:46+0000 lvl=info msg="open config file" path=/var/lib/ngrok/ngrok.yml err=nil
ngrok-1   | t=2024-06-20T03:14:46+0000 lvl=info msg="open config file" path=/var/lib/ngrok/auth-config.yml err=nil
ngrok-1   | t=2024-06-20T03:14:46+0000 lvl=info msg="starting web service" obj=web addr=0.0.0.0:4040 allow_hosts=[]
ngrok-1   | t=2024-06-20T03:14:46+0000 lvl=info msg="client session established" obj=tunnels.session
ngrok-1   | t=2024-06-20T03:14:46+0000 lvl=info msg="tunnel session started" obj=tunnels.session
ngrok-1   | t=2024-06-20T03:14:46+0000 lvl=info msg="started tunnel" obj=tunnels name=command_line addr=http://localhost:8000 url=https://94e1-223-185-128-160.ngrok-free.app
ollama-1  | time=2024-06-20T03:14:48.602Z level=INFO source=payload.go:44 msg="Dynamic LLM libraries [cpu cuda_v11]"
ollama-1  | time=2024-06-20T03:14:48.603Z level=INFO source=types.go:71 msg="inference compute" id=0 library=cpu compute="" driver=0.0 name="" total="7.7 GiB" available="3.9 GiB"
lollm-1   | INFO:     Started server process [1]
lollm-1   | INFO:     Waiting for application startup.
lollm-1   | INFO:     Application startup complete.
lollm-1   | INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
ngrok-1   | t=2024-06-20T03:16:58+0000 lvl=info msg="join connections" obj=join id=ce119162e042 l=127.0.0.1:8000 r=[2401:4900:8838:8063:f0b0:1866:e957:b3ba]:54384
lollm-1   | OLLAMA API IS http://host.docker.internal:11434/api
lollm-1   | INFO:     2401:4900:8838:8063:f0b0:1866:e957:b3ba:0 - "GET / HTTP/1.1" 200 OK

ngrok-1   | t=2024-07-09T02:34:30+0000 lvl=info msg="join connections" obj=join id=12bd50a8030b l=127.0.0.1:8000 r=18.223.220.3:44370
lollm-1   | OLLAMA API IS http://host.docker.internal:11434/api
lollm-1   | INFO:     18.223.220.3:0 - "POST /msg HTTP/1.1" 200 OK
ngrok-1   | t=2024-07-09T02:34:53+0000 lvl=info msg="join connections" obj=join id=259fda936691 l=127.0.0.1:8000 r=18.223.220.3:36712
lollm-1   | INFO:     18.223.220.3:0 - "POST /msg HTTP/1.1" 200 OK

ngrok-1   | t=2024-07-09T03:39:23+0000 lvl=info msg="join connections" obj=join id=026d8fad5c87 l=127.0.0.1:8000 r=18.223.220.3:36282
lollm-1   | setting default model 
lollm-1   | INFO:     18.223.220.3:0 - "POST /msg HTTP/1.1" 200 OK