Compare commits

..

35 Commits

Author SHA1 Message Date
Timothy Jaeryang Baek 82cda6e522 Merge pull request #5699 from open-webui/dev
Create and publish Docker images with specific build args / build-main-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-main-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / merge-main-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-cuda-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-ollama-images (push) Blocked by required conditions
0.3.29
2024-09-25 15:46:39 +02:00
Timothy J. Baek 119a7f1933 doc: changelog 2024-09-25 15:45:36 +02:00
Timothy J. Baek 6c4445d545 fix: filter outlet issue 2024-09-25 13:44:06 +02:00
Timothy J. Baek 92b1acd6fb fix: RESET_CONFIG_ON_START not working 2024-09-25 01:06:11 +02:00
Timothy J. Baek 1767b64135 fix: 'call' url search param not working 2024-09-25 01:02:48 +02:00
Timothy Jaeryang Baek e030f261d1 Merge pull request #5675 from sp301415/dev
fix: Fix KaTeX Rendering (Followup)
2024-09-24 20:58:22 +02:00
Hwang In Tak d501ece247 fix: Fix KaTeX corner cases 2024-09-25 02:15:53 +09:00
Timothy Jaeryang Baek 534e4c90ca Merge pull request #5674 from open-webui/dev
Create and publish Docker images with specific build args / build-main-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-main-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / merge-main-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-cuda-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-ollama-images (push) Blocked by required conditions
0.3.28
2024-09-24 18:52:23 +02:00
Timothy J. Baek 40f2c3521b doc: changelog 2024-09-24 18:50:47 +02:00
Timothy J. Baek 07b1327708 fix 2024-09-24 18:32:14 +02:00
Timothy J. Baek 525095b3de fix: websearch not working issue
#5672
2024-09-24 18:25:21 +02:00
Timothy Jaeryang Baek f84513e856 Merge pull request #5673 from open-webui/dev
refac: readme
2024-09-24 18:19:09 +02:00
Timothy J. Baek bf423b8577 refac: readme 2024-09-24 18:18:32 +02:00
Timothy Jaeryang Baek ba20c71963 Merge pull request #5661 from open-webui/dev
Create and publish Docker images with specific build args / build-main-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-main-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / merge-main-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-cuda-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-ollama-images (push) Blocked by required conditions
0.3.27
2024-09-24 18:13:08 +02:00
Timothy J. Baek 7bbc57f225 doc: changelog 2024-09-24 18:12:47 +02:00
Timothy J. Baek e703e172e2 chore: format 2024-09-24 18:10:14 +02:00
Timothy Jaeryang Baek 3ff52fd1ad Merge pull request #5655 from sp301415/dev
fix: Fix KaTeX Rendering
2024-09-24 17:44:08 +02:00
Timothy J. Baek e19406cdd7 fix 2024-09-24 17:43:43 +02:00
Hwang In Tak 30e65b33f6 fix: Add comments 2024-09-25 00:41:08 +09:00
Hwang In Tak 3f1255b39e fix: Change inline and block delimiters 2024-09-25 00:10:49 +09:00
Timothy J. Baek 71743b25fe chore: format 2024-09-24 16:42:42 +02:00
Timothy Jaeryang Baek c7e93b32c5 Merge pull request #5660 from open-webui/pypi-release
fix: open-webui serve
2024-09-24 16:36:37 +02:00
Timothy J. Baek 3cee507687 fix: dev2 2024-09-24 16:19:24 +02:00
Timothy J. Baek ff651ddc36 fix: dev1 2024-09-24 16:07:49 +02:00
Timothy Jaeryang Baek c0738cef26 Merge pull request #5659 from open-webui/dev
Create and publish Docker images with specific build args / build-main-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-main-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-cuda-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/amd64) (push) Waiting to run
Create and publish Docker images with specific build args / build-ollama-image (linux/arm64) (push) Waiting to run
Create and publish Docker images with specific build args / merge-main-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-cuda-images (push) Blocked by required conditions
Create and publish Docker images with specific build args / merge-ollama-images (push) Blocked by required conditions
0.3.26
2024-09-24 15:41:42 +02:00
Timothy J. Baek a44e9a8dda refac 2024-09-24 15:41:23 +02:00
Timothy J. Baek 38b9a63fa5 refac 2024-09-24 15:40:37 +02:00
Timothy J. Baek 2d60e42258 doc: changelog 2024-09-24 15:40:01 +02:00
Timothy Jaeryang Baek 60ac69eb27 Merge pull request #5656 from OriginalSimon/dev
i18n: Update Ukrainian translation
2024-09-24 15:33:15 +02:00
Timothy J. Baek 504d910557 fix: no running event loop issue 2024-09-24 15:31:55 +02:00
Hwang In Tak e48d66f918 fix: Remove unnecessary logging 2024-09-24 22:11:05 +09:00
Hwang In Tak 0bfbace9aa fix: Simplify regex 2024-09-24 22:00:01 +09:00
Simon f382a78e31 Update translation.json 2024-09-24 13:47:51 +02:00
Hwang In Tak 377cc427b6 fix: Remove unnecessary logging 2024-09-24 20:40:50 +09:00
Hwang In Tak 214546399a fix: fix katex rendering 2024-09-24 16:58:15 +09:00
13 changed files with 212 additions and 154 deletions
+28
View File
@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.3.29] - 2023-09-25
### Fixed
- **🔧 KaTeX Rendering Improvement**: Resolved specific corner cases in KaTeX rendering to enhance the display of complex mathematical notation.
- **📞 'Call' URL Parameter Fix**: Corrected functionality for 'call' URL search parameter ensuring reliable activation of voice calls through URL triggers.
- **🔄 Configuration Reset Fix**: Fixed the RESET_CONFIG_ON_START to ensure settings revert to default correctly upon each startup, improving reliability in configuration management.
- **🌍 Filter Outlet Hook Fix**: Addressed issues in the filter outlet hook, ensuring all filter functions operate as intended.
## [0.3.28] - 2024-09-24
### Fixed
- **🔍 Web Search Functionality**: Corrected an issue where the web search option was not functioning properly.
## [0.3.27] - 2024-09-24
### Fixed
- **🔄 Periodic Cleanup Error Resolved**: Fixed a critical RuntimeError related to the 'periodic_usage_pool_cleanup' coroutine, ensuring smooth and efficient performance post-pip install, correcting a persisting issue from version 0.3.26.
- **📊 Enhanced LaTeX Rendering**: Improved rendering for LaTeX content, enhancing clarity and visual presentation in documents and mathematical models.
## [0.3.26] - 2024-09-24
### Fixed
- **🔄 Event Loop Error Resolution**: Addressed a critical error where a missing running event loop caused 'periodic_usage_pool_cleanup' to fail with pip installs. This fix ensures smoother and more reliable updates and installations, enhancing overall system stability.
## [0.3.25] - 2024-09-24
### Fixed
+22 -2
View File
@@ -59,11 +59,31 @@ Don't forget to explore our sibling project, [Open WebUI Community](https://open
## How to Install 🚀
> [!NOTE]
> Please note that for certain Docker environments, additional configurations might be needed. If you encounter any connection issues, our detailed guide on [Open WebUI Documentation](https://docs.openwebui.com/) is ready to assist you.
### Installation via Python pip 🐍
Open WebUI can be installed using pip, the Python package installer. Before proceeding, ensure you're using **Python 3.11** to avoid compatibility issues.
1. **Install Open WebUI**:
Open your terminal and run the following command to install Open WebUI:
```bash
pip install open-webui
```
2. **Running Open WebUI**:
After installation, you can start Open WebUI by executing:
```bash
open-webui serve
```
This will start the Open WebUI server, which you can access at [http://localhost:8080](http://localhost:8080)
### Quick Start with Docker 🐳
> [!NOTE]
> Please note that for certain Docker environments, additional configurations might be needed. If you encounter any connection issues, our detailed guide on [Open WebUI Documentation](https://docs.openwebui.com/) is ready to assist you.
> [!WARNING]
> When using Docker to install Open WebUI, make sure to include the `-v open-webui:/app/backend/data` in your Docker command. This step is crucial as it ensures your database is properly mounted and prevents any loss of data.
+1
View File
@@ -52,6 +52,7 @@ def serve(
)
os.environ["USE_CUDA_DOCKER"] = "false"
os.environ["LD_LIBRARY_PATH"] = ":".join(LD_LIBRARY_PATH)
import open_webui.main # we need set environment variables before importing main
uvicorn.run(open_webui.main.app, host=host, port=port, forwarded_allow_ips="*")
+4 -4
View File
@@ -48,8 +48,6 @@ else:
)
app = socketio.ASGIApp(sio, socketio_path="/ws/socket.io")
# Dictionary to maintain the user pool
if WEBSOCKET_MANAGER == "redis":
@@ -92,8 +90,10 @@ async def periodic_usage_pool_cleanup():
await asyncio.sleep(TIMEOUT_DURATION)
# Start the cleanup task when your app starts
asyncio.create_task(periodic_usage_pool_cleanup())
app = socketio.ASGIApp(
sio,
socketio_path="/ws/socket.io",
)
def get_models_in_use():
+6
View File
@@ -87,6 +87,12 @@ def save_to_db(data):
db.commit()
def reset_config():
with get_db() as db:
db.query(Config).delete()
db.commit()
# When initializing, check if config.json exists and migrate it to the database
if os.path.exists(f"{DATA_DIR}/config.json"):
data = load_json_config()
+4 -12
View File
@@ -234,18 +234,6 @@ if FROM_INIT_PY:
).resolve()
RESET_CONFIG_ON_START = (
os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true"
)
if RESET_CONFIG_ON_START:
try:
os.remove(f"{DATA_DIR}/config.json")
with open(f"{DATA_DIR}/config.json", "w") as f:
f.write("{}")
except Exception:
pass
####################################
# Database
####################################
@@ -265,6 +253,10 @@ if "postgres://" in DATABASE_URL:
DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://")
RESET_CONFIG_ON_START = (
os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true"
)
####################################
# WEBUI_AUTH (Required for security)
####################################
+10 -2
View File
@@ -8,6 +8,8 @@ import shutil
import sys
import time
import uuid
import asyncio
from contextlib import asynccontextmanager
from typing import Optional
@@ -31,7 +33,7 @@ from open_webui.apps.openai.main import (
from open_webui.apps.openai.main import get_all_models as get_openai_models
from open_webui.apps.rag.main import app as rag_app
from open_webui.apps.rag.utils import get_rag_context, rag_template
from open_webui.apps.socket.main import app as socket_app
from open_webui.apps.socket.main import app as socket_app, periodic_usage_pool_cleanup
from open_webui.apps.socket.main import get_event_call, get_event_emitter
from open_webui.apps.webui.internal.db import Session
from open_webui.apps.webui.main import app as webui_app
@@ -77,6 +79,7 @@ from open_webui.config import (
WEBUI_NAME,
AppConfig,
run_migrations,
reset_config,
)
from open_webui.constants import ERROR_MESSAGES, TASKS, WEBHOOK_MESSAGES
from open_webui.env import (
@@ -90,6 +93,7 @@ from open_webui.env import (
WEBUI_SESSION_COOKIE_SAME_SITE,
WEBUI_SESSION_COOKIE_SECURE,
WEBUI_URL,
RESET_CONFIG_ON_START,
)
from fastapi import (
Depends,
@@ -184,6 +188,11 @@ https://github.com/open-webui/open-webui
@asynccontextmanager
async def lifespan(app: FastAPI):
run_migrations()
if RESET_CONFIG_ON_START:
reset_config()
asyncio.create_task(periodic_usage_pool_cleanup())
yield
@@ -851,7 +860,6 @@ async def inspect_websocket(request: Request, call_next):
app.mount("/ws", socket_app)
app.mount("/ollama", ollama_app)
app.mount("/openai", openai_app)
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "open-webui",
"version": "0.3.25",
"version": "0.3.29",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "open-webui",
"version": "0.3.25",
"version": "0.3.29",
"dependencies": {
"@codemirror/lang-javascript": "^6.2.2",
"@codemirror/lang-python": "^6.1.6",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "open-webui",
"version": "0.3.25",
"version": "0.3.29",
"private": true,
"scripts": {
"dev": "npm run pyodide:fetch && vite dev --host",
+26 -19
View File
@@ -357,6 +357,7 @@
if ($page.url.searchParams.get('call') === 'true') {
showCallOverlay.set(true);
showControls.set(true);
}
selectedModels = selectedModels.map((modelId) =>
@@ -482,11 +483,12 @@
}
}
await tick();
if ($chatId == chatId) {
if (!$temporaryChatEnabled) {
chat = await updateChatById(localStorage.token, chatId, {
models: selectedModels,
messages: messages,
history: history,
params: params,
files: chatFiles
@@ -1133,13 +1135,6 @@
}
}
}
await chatCompletedHandler(
_chatId,
model.id,
responseMessageId,
createMessagesList(responseMessageId)
);
} else {
if (res !== null) {
const error = await res.json();
@@ -1168,11 +1163,18 @@
(status) => status.action !== 'knowledge_search'
);
}
history.messages[responseMessageId] = responseMessage;
}
await saveChatHandler(_chatId);
history.messages[responseMessageId] = responseMessage;
await chatCompletedHandler(
_chatId,
model.id,
responseMessageId,
createMessagesList(responseMessageId)
);
stopResponseFlag = false;
await tick();
@@ -1429,13 +1431,6 @@
}
}
await chatCompletedHandler(
_chatId,
model.id,
responseMessageId,
createMessagesList(responseMessageId)
);
if ($settings.notificationEnabled && !document.hasFocus()) {
const notification = new Notification(`${model.id}`, {
body: responseMessage.content,
@@ -1463,6 +1458,13 @@
history.messages[responseMessageId] = responseMessage;
await chatCompletedHandler(
_chatId,
model.id,
responseMessageId,
createMessagesList(responseMessageId)
);
stopResponseFlag = false;
await tick();
@@ -1683,9 +1685,14 @@
}
};
const getWebSearchResults = async (model: string, parentId: string, responseId: string) => {
const responseMessage = history.messages[responseId];
const getWebSearchResults = async (
model: string,
parentId: string,
responseMessageId: string
) => {
const responseMessage = history.messages[responseMessageId];
const userMessage = history.messages[parentId];
const messages = createMessagesList(history.currentId);
responseMessage.statusHistory = [
{
+17 -17
View File
@@ -9,7 +9,7 @@
"{{user}}'s Chats": "Чати {{user}}а",
"{{webUIName}} Backend Required": "Необхідно підключення бекенду {{webUIName}}",
"*Prompt node ID(s) are required for image generation": "*Для генерації зображення потрібно вказати ідентифікатор(и) вузла(ів)",
"A new version (v{{LATEST_VERSION}}) is now available.": "",
"A new version (v{{LATEST_VERSION}}) is now available.": "Нова версія (в{{LATEST_VERSION}}) зараз доступна.",
"A task model is used when performing tasks such as generating titles for chats and web search queries": "Модель задач використовується при виконанні таких завдань, як генерація заголовків для чатів та пошукових запитів в Інтернеті",
"a user": "користувача",
"About": "Про програму",
@@ -74,10 +74,10 @@
"AUTOMATIC1111 Api Auth String": "AUTOMATIC1111 Рядок авторизації API",
"AUTOMATIC1111 Base URL": "URL-адреса AUTOMATIC1111",
"AUTOMATIC1111 Base URL is required.": "Необхідна URL-адреса AUTOMATIC1111.",
"Available list": "",
"Available list": "Список доступності",
"available!": "доступно!",
"Azure AI Speech": "",
"Azure Region": "",
"Azure AI Speech": "Мовлення Azure AI",
"Azure Region": "Регіон Azure",
"Back": "Назад",
"Bad Response": "Неправильна відповідь",
"Banners": "Прапори",
@@ -98,7 +98,7 @@
"Chat Bubble UI": "Чат у вигляді бульбашок",
"Chat Controls": "Керування чатом",
"Chat direction": "Напрям чату",
"Chat Overview": "",
"Chat Overview": "Огляд чату",
"Chats": "Чати",
"Check Again": "Перевірити ще раз",
"Check for updates": "Перевірити оновлення",
@@ -416,8 +416,8 @@
"Model {{modelId}} not found": "Модель {{modelId}} не знайдено",
"Model {{modelName}} is not vision capable": "Модель {{modelName}} не здатна бачити",
"Model {{name}} is now {{status}}": "Модель {{name}} тепер має {{status}}",
"Model {{name}} is now at the top": "",
"Model accepts image inputs": "",
"Model {{name}} is now at the top": "Модель {{name}} тепер на першому місці",
"Model accepts image inputs": "Модель приймає зображеня",
"Model created successfully!": "Модель створено успішно!",
"Model filesystem path detected. Model shortname is required for update, cannot continue.": "Виявлено шлях до файлової системи моделі. Для оновлення потрібно вказати коротке ім'я моделі, не вдасться продовжити.",
"Model ID": "ID моделі",
@@ -429,7 +429,7 @@
"Modelfile Content": "Вміст файлу моделі",
"Models": "Моделі",
"More": "Більше",
"Move to Top": "",
"Move to Top": "Перейти до початку",
"Name": "Ім'я",
"Name Tag": "Назва тегу",
"Name your model": "Назвіть свою модель",
@@ -466,7 +466,7 @@
"Oops! Looks like the URL is invalid. Please double-check and try again.": "Упс! Схоже, що URL-адреса невірна. Будь ласка, перевірте ще раз та спробуйте ще раз.",
"Oops! There was an error in the previous response. Please try again or contact admin.": "Упс! У попередній відповіді сталася помилка. Будь ласка, спробуйте ще раз або зверніться до адміністратора.",
"Oops! You're using an unsupported method (frontend only). Please serve the WebUI from the backend.": "Упс! Ви використовуєте непідтримуваний метод (тільки для фронтенду). Будь ласка, обслуговуйте WebUI з бекенду.",
"Open file": "",
"Open file": "Відкрити файл",
"Open new chat": "Відкрити новий чат",
"Open WebUI version (v{{OPEN_WEBUI_VERSION}}) is lower than required version (v{{REQUIRED_VERSION}})": "Open WebUI версія (v{{OPEN_WEBUI_VERSION}}) нижча за необхідну версію (v{{REQUIRED_VERSION}})",
"OpenAI": "OpenAI",
@@ -476,9 +476,9 @@
"OpenAI URL/Key required.": "Потрібен OpenAI URL/ключ.",
"or": "або",
"Other": "Інше",
"Output format": "",
"Overview": "",
"page": "",
"Output format": "Формат відповіді",
"Overview": "Огляд",
"page": "сторінка",
"Password": "Пароль",
"PDF document (.pdf)": "PDF документ (.pdf)",
"PDF Extract Images (OCR)": "Розпізнавання зображень з PDF (OCR)",
@@ -542,7 +542,7 @@
"Save": "Зберегти",
"Save & Create": "Зберегти та створити",
"Save & Update": "Зберегти та оновити",
"Save As Copy": "",
"Save As Copy": "Зберегти як копію",
"Save Tag": "Зберегти тег",
"Saving chat logs directly to your browser's storage is no longer supported. Please take a moment to download and delete your chat logs by clicking the button below. Don't worry, you can easily re-import your chat logs to the backend through": "Збереження журналів чату безпосередньо в сховище вашого браузера більше не підтримується. Будь ласка, завантажте та видаліть журнали чату, натиснувши кнопку нижче. Не хвилюйтеся, ви можете легко повторно імпортувати журнали чату до бекенду через",
"Scan": "Сканування",
@@ -587,7 +587,7 @@
"Send": "Надіслати",
"Send a Message": "Надіслати повідомлення",
"Send message": "Надіслати повідомлення",
"Sends `stream_options: { include_usage: true }` in the request.\nSupported providers will return token usage information in the response when set.": "",
"Sends `stream_options: { include_usage: true }` in the request.\nSupported providers will return token usage information in the response when set.": "Відправляє `stream_options: { include_usage: true }` у запиті.\nПідтримувані постачальники повернуть інформацію про використання токену у відповіді, якщо вона встановлена.",
"September": "Вересень",
"Serper API Key": "Ключ API Serper",
"Serply API Key": "Ключ API Serply",
@@ -621,11 +621,11 @@
"Sign up": "Зареєструватися",
"Signing in": "Увійдіть в систему",
"Source": "Джерело",
"Speech Playback Speed": "",
"Speech Playback Speed": "Швидкість відтворення мовлення",
"Speech recognition error: {{error}}": "Помилка розпізнавання мови: {{error}}",
"Speech-to-Text Engine": "Система розпізнавання мови",
"Stop Sequence": "Символ зупинки",
"Stream Chat Response": "",
"Stream Chat Response": "Відповідь стрім-чату",
"STT Model": "Модель STT ",
"STT Settings": "Налаштування STT",
"Submit": "Надіслати",
@@ -705,7 +705,7 @@
"Unpin": "Відчепити",
"Update": "Оновлення",
"Update and Copy Link": "Оновлення та копіювання посилання",
"Update for the latest features and improvements.": "",
"Update for the latest features and improvements.": "Оновіть програми для нових функцій та покращень.",
"Update password": "Оновити пароль",
"Updated at": "Оновлено на",
"Upload": "Завантажити",
-18
View File
@@ -8,23 +8,6 @@ import { TTS_RESPONSE_SPLIT } from '$lib/types';
// Helper functions
//////////////////////////
const convertLatexToSingleLine = (content) => {
// Patterns to match multiline LaTeX blocks
const patterns = [
/(\$\$\s[\s\S]*?\s\$\$)/g, // Match $$ ... $$
/(\\\[[\s\S]*?\\\])/g, // Match \[ ... \]
/(\\begin\{[a-z]+\}[\s\S]*?\\end\{[a-z]+\})/g // Match \begin{...} ... \end{...}
];
patterns.forEach((pattern) => {
content = content.replace(pattern, (match) => {
return match.replace(/\s*\n\s*/g, ' ').trim();
});
});
return content;
};
export const replaceTokens = (content, char, user) => {
const charToken = /{{char}}/gi;
const userToken = /{{user}}/gi;
@@ -68,7 +51,6 @@ export const sanitizeResponseContent = (content: string) => {
};
export const processResponseContent = (content: string) => {
content = convertLatexToSingleLine(content);
return content.trim();
};
+91 -77
View File
@@ -1,14 +1,13 @@
import katex from 'katex';
const DELIMITER_LIST = [
{ left: '$$', right: '$$', display: false },
{ left: '$$', right: '$$', display: true },
{ left: '$', right: '$', display: false },
{ left: '\\pu{', right: '}', display: false },
{ left: '\\ce{', right: '}', display: false },
{ left: '\\(', right: '\\)', display: false },
{ left: '( ', right: ' )', display: false },
{ left: '\\[', right: '\\]', display: true },
{ left: '[ ', right: ' ]', display: true }
{ left: '\\begin{equation}', right: '\\end{equation}', display: true }
];
// const DELIMITER_LIST = [
@@ -28,24 +27,27 @@ function escapeRegex(string) {
function generateRegexRules(delimiters) {
delimiters.forEach((delimiter) => {
const { left, right } = delimiter;
const { left, right, display } = delimiter;
// Ensure regex-safe delimiters
const escapedLeft = escapeRegex(left);
const escapedRight = escapeRegex(right);
// Inline pattern - Capture group $1, token content, followed by end delimiter and normal punctuation marks.
// Example: $text$
inlinePatterns.push(
`${escapedLeft}((?:\\\\.|[^\\\\\\n])*?(?:\\\\.|[^\\\\\\n${escapedRight}]))${escapedRight}`
);
// Block pattern - Starts and ends with the delimiter on new lines. Example:
// $$\ncontent here\n$$
blockPatterns.push(`${escapedLeft}\n((?:\\\\[^]|[^\\\\])+?)\n${escapedRight}`);
if (!display) {
// For inline delimiters, we match everyting
inlinePatterns.push(`${escapedLeft}((?:\\\\[^]|[^\\\\])+?)${escapedRight}`);
} else {
// Block delimiters doubles as inline delimiters when not followed by a newline
inlinePatterns.push(`${escapedLeft}(?!\\n)((?:\\\\[^]|[^\\\\])+?)(?!\\n)${escapedRight}`);
blockPatterns.push(`${escapedLeft}\\n((?:\\\\[^]|[^\\\\])+?)\\n${escapedRight}`);
}
});
const inlineRule = new RegExp(`^(${inlinePatterns.join('|')})(?=[\\s?!.,:?!。,:]|$)`, 'u');
const blockRule = new RegExp(`^(${blockPatterns.join('|')})(?:\n|$)`, 'u');
// Math formulas can end in special characters
const inlineRule = new RegExp(
`^(${inlinePatterns.join('|')})(?=[\\s?。,!-\/:-@[-\`{-~]|$)`,
'u'
);
const blockRule = new RegExp(`^(${blockPatterns.join('|')})(?=[\\s?。,!-\/:-@[-\`{-~]|$)`, 'u');
return { inlineRule, blockRule };
}
@@ -54,85 +56,97 @@ const { inlineRule, blockRule } = generateRegexRules(DELIMITER_LIST);
export default function (options = {}) {
return {
extensions: [
inlineKatex(options, createRenderer(options, false)),
blockKatex(options, createRenderer(options, true))
]
extensions: [inlineKatex(options), blockKatex(options)]
};
}
function createRenderer(options, newlineAfter) {
return (token) =>
katex.renderToString(token.text, { ...options, displayMode: token.displayMode }) +
(newlineAfter ? '\n' : '');
function katexStart(src, displayMode: boolean) {
let ruleReg = displayMode ? blockRule : inlineRule;
let indexSrc = src;
while (indexSrc) {
let index = -1;
let startIndex = -1;
let startDelimiter = '';
let endDelimiter = '';
for (let delimiter of DELIMITER_LIST) {
if (delimiter.display !== displayMode) {
continue;
}
startIndex = indexSrc.indexOf(delimiter.left);
if (startIndex === -1) {
continue;
}
index = startIndex;
startDelimiter = delimiter.left;
endDelimiter = delimiter.right;
}
if (index === -1) {
return;
}
// Check if the delimiter is preceded by a special character.
// If it does, then it's potentially a math formula.
const f = index === 0 || indexSrc.charAt(index - 1).match(/[\s?。,!-\/:-@[-`{-~]/);
if (f) {
const possibleKatex = indexSrc.substring(index);
if (possibleKatex.match(ruleReg)) {
return index;
}
}
indexSrc = indexSrc.substring(index + startDelimiter.length).replace(endDelimiter, '');
}
}
function inlineKatex(options, renderer) {
const ruleReg = inlineRule;
function katexTokenizer(src, tokens, displayMode: boolean) {
let ruleReg = displayMode ? blockRule : inlineRule;
let type = displayMode ? 'blockKatex' : 'inlineKatex';
const match = src.match(ruleReg);
if (match) {
const text = match
.slice(2)
.filter((item) => item)
.find((item) => item.trim());
return {
type,
raw: match[0],
text: text,
displayMode
};
}
}
function inlineKatex(options) {
return {
name: 'inlineKatex',
level: 'inline',
start(src) {
let index;
let indexSrc = src;
while (indexSrc) {
index = indexSrc.indexOf('$');
if (index === -1) {
return;
}
const f = index === 0 || indexSrc.charAt(index - 1) === ' ';
if (f) {
const possibleKatex = indexSrc.substring(index);
if (possibleKatex.match(ruleReg)) {
return index;
}
}
indexSrc = indexSrc.substring(index + 1).replace(/^\$+/, '');
}
return katexStart(src, false);
},
tokenizer(src, tokens) {
const match = src.match(ruleReg);
if (match) {
const text = match
.slice(2)
.filter((item) => item)
.find((item) => item.trim());
return {
type: 'inlineKatex',
raw: match[0],
text: text
};
}
},
renderer
return katexTokenizer(src, tokens, false);
}
};
}
function blockKatex(options, renderer) {
function blockKatex(options) {
return {
name: 'blockKatex',
level: 'block',
tokenizer(src, tokens) {
const match = src.match(blockRule);
if (match) {
const text = match
.slice(2)
.filter((item) => item)
.find((item) => item.trim());
return {
type: 'blockKatex',
raw: match[0],
text: text
};
}
start(src) {
return katexStart(src, true);
},
renderer
tokenizer(src, tokens) {
return katexTokenizer(src, tokens, true);
}
};
}