code:
import { config } from "../../../config";
import { logger } from "../../../logger";
import { ExpressHttpProxyReqCallback, isCompletionRequest } from ".";

const MAX_TOKENS = config.maxOutputTokens;

/** Enforce a maximum number of tokens requested from OpenAI. */
export const limitOutputTokens: ExpressHttpProxyReqCallback = (
  _proxyReq,
  req
) => {
  if (isCompletionRequest(req) && req.body?.max_tokens) {
    // convert bad or missing input to a MAX_TOKENS
    if (typeof req.body.max_tokens !== "number") {
      logger.warn(
        `Invalid max_tokens value: ${req.body.max_tokens}. Using ${MAX_TOKENS}`
      );
      req.body.max_tokens = MAX_TOKENS;
    }

    const originalTokens = req.body.max_tokens;
    req.body.max_tokens = Math.min(req.body.max_tokens, MAX_TOKENS);
    if (originalTokens !== req.body.max_tokens) {
      logger.warn(
        `Limiting max_tokens from ${originalTokens} to ${req.body.max_tokens}`
      );
    }
  }
};


code:
import { Response } from "express";
import * as http from "http";
import { RawResponseBodyHandler, decodeResponseBody } from ".";

/**
 * Consume the SSE stream and forward events to the client. Once the stream is
 * stream is closed, resolve with the full response body so that subsequent
 * middleware can work with it.
 *
 * Typically we would only need of the raw response handlers to execute, but
 * in the event a streamed request results in a non-200 response, we need to
 * fall back to the non-streaming response handler so that the error handler
 * can inspect the error response.
 */
export const handleStreamedResponse: RawResponseBodyHandler = async (
  proxyRes,
  req,
  res
) => {
  if (!req.isStreaming) {
    req.log.error(
      { api: req.api, key: req.key?.hash },
      `handleEventSource called for non-streaming request, which isn't valid.`
    );
    throw new Error("handleEventSource called for non-streaming request.");
  }

  if (proxyRes.statusCode !== 200) {
    // Ensure we use the non-streaming middleware stack since we won't be
    // getting any events.
    req.isStreaming = false;
    req.log.warn(
      `Streaming request to ${req.api} returned ${proxyRes.statusCode} status code. Falling back to non-streaming response handler.`
    );
    return decodeResponseBody(proxyRes, req, res);
  }

  return new Promise((resolve, reject) => {
    req.log.info(
      { api: req.api, key: req.key?.hash },
      `Starting to proxy SSE stream.`
    );

    // Queued streaming requests will already have a connection open and headers
    // sent due to the heartbeat handler.  In that case we can just start
    // streaming the response without sending headers.
    if (!res.headersSent) {
      res.setHeader("Content-Type", "text/event-stream");
      res.setHeader("Cache-Control", "no-cache");
      res.setHeader("Connection", "keep-alive");
      res.setHeader("X-Accel-Buffering", "no");
      copyHeaders(proxyRes, res);
      res.flushHeaders();
    }

    const chunks: Buffer[] = [];
    proxyRes.on("data", (chunk) => {
      chunks.push(chunk);
      res.write(chunk);
    });

    proxyRes.on("end", () => {
      const finalBody = convertEventsToOpenAiResponse(chunks);
      req.log.info(
        { api: req.api, key: req.key?.hash },
        `Finished proxying SSE stream.`
      );
      res.end();
      resolve(finalBody);
    });
    proxyRes.on("error", (err) => {
      req.log.error(
        { error: err, api: req.api, key: req.key?.hash },
        `Error while streaming response.`
      );
      // OAI's spec doesn't allow for error events and clients wouldn't know
      // what to do with them anyway, so we'll just send a completion event
      // with the error message.
      const fakeErrorEvent = {
        id: "chatcmpl-error",
        object: "chat.completion.chunk",
        created: Date.now(),
        model: "",
        choices: [
          {
            delta: { content: "[Proxy streaming error: " + err.message + "]" },
            index: 0,
            finish_reason: "error",
          },
        ],
      };
      res.write(`data: ${JSON.stringify(fakeErrorEvent)}\n\n`);
      res.write("data: [DONE]\n\n");
      res.end();
      reject(err);
    });
  });
};

/** Copy headers, excluding ones we're already setting for the SSE response. */
const copyHeaders = (proxyRes: http.IncomingMessage, res: Response) => {
  const toOmit = [
    "content-length",
    "content-encoding",
    "transfer-encoding",
    "content-type",
    "connection",
    "cache-control",
  ];
  for (const [key, value] of Object.entries(proxyRes.headers)) {
    if (!toOmit.includes(key) && value) {
      res.setHeader(key, value);
    }
  }
};

type OpenAiChatCompletionResponse = {
  id: string;
  object: string;
  created: number;
  model: string;
  choices: {
    message: { role: string; content: string };
    finish_reason: string | null;
    index: number;
  }[];
};

/** Converts the event stream chunks into a single completion response. */
const convertEventsToOpenAiResponse = (chunks: Buffer[]) => {
  let response: OpenAiChatCompletionResponse = {
    id: "",
    object: "",
    created: 0,
    model: "",
    choices: [],
  };
  const events = Buffer.concat(chunks)
    .toString()
    .trim()
    .split("\n\n")
    .map((line) => line.trim());

  response = events.reduce((acc, chunk, i) => {
    if (!chunk.startsWith("data: ")) {
      return acc;
    }

    if (chunk === "data: [DONE]") {
      return acc;
    }

    const data = JSON.parse(chunk.slice("data: ".length));
    if (i === 0) {
      return {
        id: data.id,
        object: data.object,
        created: data.created,
        model: data.model,
        choices: [
          {
            message: { role: data.choices[0].delta.role, content: "" },
            index: 0,
            finish_reason: null,
          },
        ],
      };
    }

    if (data.choices[0].delta.content) {
      acc.choices[0].message.content += data.choices[0].delta.content;
    }
    acc.choices[0].finish_reason = data.choices[0].finish_reason;
    return acc;
  }, response);
  return response;
};

repo_name: TheRealZoink/oai

license: unknown


code:
import { Request, Response } from "express";
import * as http from "http";
import util from "util";
import zlib from "zlib";
import * as httpProxy from "http-proxy";
import { config } from "../../../config";
import { logger } from "../../../logger";
import { keyPool } from "../../../key-management";
import { buildFakeSseMessage, enqueue, trackWaitTime } from "../../queue";
import { handleStreamedResponse } from "./handle-streamed-response";
import { logPrompt } from "./log-prompt";
import { incrementPromptCount } from "../../auth/user-store";

export const QUOTA_ROUTES = ["/v1/chat/completions"];
const DECODER_MAP = {
  gzip: util.promisify(zlib.gunzip),
  deflate: util.promisify(zlib.inflate),
  br: util.promisify(zlib.brotliDecompress),
};

const isSupportedContentEncoding = (
  contentEncoding: string
): contentEncoding is keyof typeof DECODER_MAP => {
  return contentEncoding in DECODER_MAP;
};

class RetryableError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "RetryableError";
  }
}

/**
 * Either decodes or streams the entire response body and then passes it as the
 * last argument to the rest of the middleware stack.
 */
export type RawResponseBodyHandler = (
  proxyRes: http.IncomingMessage,
  req: Request,
  res: Response
) => Promise<string | Record<string, any>>;
export type ProxyResHandlerWithBody = (
  proxyRes: http.IncomingMessage,
  req: Request,
  res: Response,
  /**
   * This will be an object if the response content-type is application/json,
   * or if the response is a streaming response. Otherwise it will be a string.
   */
  body: string | Record<string, any>
) => Promise<void>;
export type ProxyResMiddleware = ProxyResHandlerWithBody[];

/**
 * Returns a on.proxyRes handler that executes the given middleware stack after
 * the common proxy response handlers have processed the response and decoded
 * the body.  Custom middleware won't execute if the response is determined to
 * be an error from the upstream service as the response will be taken over by
 * the common error handler.
 *
 * For streaming responses, the handleStream middleware will block remaining
 * middleware from executing as it consumes the stream and forwards events to
 * the client. Once the stream is closed, the finalized body will be attached
 * to res.body and the remaining middleware will execute.
 */
export const createOnProxyResHandler = (apiMiddleware: ProxyResMiddleware) => {
  return async (
    proxyRes: http.IncomingMessage,
    req: Request,
    res: Response
  ) => {
    const initialHandler = req.isStreaming
      ? handleStreamedResponse
      : decodeResponseBody;

    let lastMiddlewareName = initialHandler.name;

    try {
      const body = await initialHandler(proxyRes, req, res);

      const middlewareStack: ProxyResMiddleware = [];

      if (req.isStreaming) {
        // `handleStreamedResponse` writes to the response and ends it, so
        // we can only execute middleware that doesn't write to the response.
        middlewareStack.push(trackRateLimit, incrementKeyUsage, logPrompt);
      } else {
        middlewareStack.push(
          trackRateLimit,
          handleUpstreamErrors,
          incrementKeyUsage,
          copyHttpHeaders,
          logPrompt,
          ...apiMiddleware
        );
      }

      for (const middleware of middlewareStack) {
        lastMiddlewareName = middleware.name;
        await middleware(proxyRes, req, res, body);
      }

      trackWaitTime(req);
    } catch (error: any) {
      // Hack: if the error is a retryable rate-limit error, the request has
      // been re-enqueued and we can just return without doing anything else.
      if (error instanceof RetryableError) {
        return;
      }

      const errorData = {
        error: error.stack,
        thrownBy: lastMiddlewareName,
        key: req.key?.hash,
      };
      const message = `Error while executing proxy response middleware: ${lastMiddlewareName} (${error.message})`;
      if (res.headersSent) {
        req.log.error(errorData, message);
        // This should have already been handled by the error handler, but
        // just in case...
        if (!res.writableEnded) {
          res.end();
        }
        return;
      }
      logger.error(errorData, message);
      res
        .status(500)
        .json({ error: "Internal server error", proxy_note: message });
    }
  };
};

function reenqueueRequest(req: Request) {
  req.log.info(
    { key: req.key?.hash, retryCount: req.retryCount },
    `Re-enqueueing request due to rate-limit error`
  );
  req.retryCount++;
  enqueue(req);
}

/**
 * Handles the response from the upstream service and decodes the body if
 * necessary.  If the response is JSON, it will be parsed and returned as an
 * object.  Otherwise, it will be returned as a string.
 * @throws {Error} Unsupported content-encoding or invalid application/json body
 */
export const decodeResponseBody: RawResponseBodyHandler = async (
  proxyRes,
  req,
  res
) => {
  if (req.isStreaming) {
    req.log.error(
      { api: req.api, key: req.key?.hash },
      `decodeResponseBody called for a streaming request, which isn't valid.`
    );
    throw new Error("decodeResponseBody called for a streaming request.");
  }

  const promise = new Promise<string>((resolve, reject) => {
    let chunks: Buffer[] = [];
    proxyRes.on("data", (chunk) => chunks.push(chunk));
    proxyRes.on("end", async () => {
      let body = Buffer.concat(chunks);

      const contentEncoding = proxyRes.headers["content-encoding"];
      if (contentEncoding) {
        if (isSupportedContentEncoding(contentEncoding)) {
          const decoder = DECODER_MAP[contentEncoding];
          body = await decoder(body);
        } else {
          const errorMessage = `Proxy received response with unsupported content-encoding: ${contentEncoding}`;
          logger.warn({ contentEncoding, key: req.key?.hash }, errorMessage);
          writeErrorResponse(res, 500, {
            error: errorMessage,
            contentEncoding,
          });
          return reject(errorMessage);
        }
      }

      try {
        if (proxyRes.headers["content-type"]?.includes("application/json")) {
          const json = JSON.parse(body.toString());
          return resolve(json);
        }
        return resolve(body.toString());
      } catch (error: any) {
        const errorMessage = `Proxy received response with invalid JSON: ${error.message}`;
        logger.warn({ error, key: req.key?.hash }, errorMessage);
        writeErrorResponse(res, 500, { error: errorMessage });
        return reject(errorMessage);
      }
    });
  });
  return promise;
};

// TODO: This is too specific to OpenAI's error responses, Anthropic errors
// will need a different handler.
/**
 * Handles non-2xx responses from the upstream service.  If the proxied response
 * is an error, this will respond to the client with an error payload and throw
 * an error to stop the middleware stack.
 * On 429 errors, if request queueing is enabled, the request will be silently
 * re-enqueued.  Otherwise, the request will be rejected with an error payload.
 * @throws {Error} On HTTP error status code from upstream service
 */
const handleUpstreamErrors: ProxyResHandlerWithBody = async (
  proxyRes,
  req,
  res,
  body
) => {
  const statusCode = proxyRes.statusCode || 500;

  if (statusCode < 400) {
    return;
  }

  let errorPayload: Record<string, any>;
  // Subtract 1 from available keys because if this message is being shown,
  // it's because the key is about to be disabled.
  const availableKeys = keyPool.available() - 1;
  const tryAgainMessage = Boolean(availableKeys)
    ? `There are ${availableKeys} more keys available; try your request again.`
    : "There are no more keys available.";

  try {
    if (typeof body === "object") {
      errorPayload = body;
    } else {
      throw new Error("Received unparsable error response from upstream.");
    }
  } catch (parseError: any) {
    const statusMessage = proxyRes.statusMessage || "Unknown error";
    // Likely Bad Gateway or Gateway Timeout from OpenAI's Cloudflare proxy
    logger.warn(
      { statusCode, statusMessage, key: req.key?.hash },
      parseError.message
    );

    const errorObject = {
      statusCode,
      statusMessage: proxyRes.statusMessage,
      error: parseError.message,
      proxy_note: `This is likely a temporary error with the upstream service.`,
    };
    writeErrorResponse(res, statusCode, errorObject);
    throw new Error(parseError.message);
  }

  logger.warn(
    {
      statusCode,
      type: errorPayload.error?.code,
      errorPayload,
      key: req.key?.hash,
    },
    `Received error response from upstream. (${proxyRes.statusMessage})`
  );

  if (statusCode === 400) {
    // Bad request (likely prompt is too long)
    errorPayload.proxy_note = `OpenAI rejected the request as invalid. Your prompt may be too long for ${req.body?.model}.`;
  } else if (statusCode === 401) {
    // Key is invalid or was revoked
    keyPool.disable(req.key!);
    errorPayload.proxy_note = `The OpenAI key is invalid or revoked. ${tryAgainMessage}`;
  } else if (statusCode === 429) {
    const type = errorPayload.error?.type;
    if (type === "insufficient_quota") {
      // Billing quota exceeded (key is dead, disable it)
      keyPool.disable(req.key!);
      errorPayload.proxy_note = `Assigned key's quota has been exceeded. ${tryAgainMessage}`;
    } else if (type === "billing_not_active") {
      // Billing is not active (key is dead, disable it)
      keyPool.disable(req.key!);
      errorPayload.proxy_note = `Assigned key was deactivated by OpenAI. ${tryAgainMessage}`;
    } else if (type === "requests" || type === "tokens") {
      // Per-minute request or token rate limit is exceeded, which we can retry
      keyPool.markRateLimited(req.key!.hash);
      if (config.queueMode !== "none") {
        reenqueueRequest(req);
        // TODO: I don't like using an error to control flow here
        throw new RetryableError("Rate-limited request re-enqueued.");
      }
      errorPayload.proxy_note = `Assigned key's '${type}' rate limit has been exceeded. Try again later.`;
    } else {
      // OpenAI probably overloaded
      errorPayload.proxy_note = `This is likely a temporary error with OpenAI. Try again in a few seconds.`;
    }
  } else if (statusCode === 404) {
    // Most likely model not found
    // TODO: this probably doesn't handle GPT-4-32k variants properly if the
    // proxy has keys for both the 8k and 32k context models at the same time.
    if (errorPayload.error?.code === "model_not_found") {
      if (req.key!.isGpt4) {
        errorPayload.proxy_note = `Assigned key isn't provisioned for the GPT-4 snapshot you requested. Try again to get a different key, or use Turbo.`;
      } else {
        errorPayload.proxy_note = `No model was found for this key.`;
      }
    }
  } else {
    errorPayload.proxy_note = `Unrecognized error from OpenAI.`;
  }

  // Some OAI errors contain the organization ID, which we don't want to reveal.
  if (errorPayload.error?.message) {
    errorPayload.error.message = errorPayload.error.message.replace(
      /org-.{24}/gm,
      "org-xxxxxxxxxxxxxxxxxxx"
    );
  }

  writeErrorResponse(res, statusCode, errorPayload);
  throw new Error(errorPayload.error?.message);
};

function writeErrorResponse(
  res: Response,
  statusCode: number,
  errorPayload: Record<string, any>
) {
  // If we're mid-SSE stream, send a data event with the error payload and end
  // the stream. Otherwise just send a normal error response.
  if (
    res.headersSent ||
    res.getHeader("content-type") === "text/event-stream"
  ) {
    const msg = buildFakeSseMessage(
      `upstream error (${statusCode})`,
      JSON.stringify(errorPayload, null, 2)
    );
    res.write(msg);
    res.write(`data: [DONE]\n\n`);
    res.end();
  } else {
    res.status(statusCode).json(errorPayload);
  }
}

/** Handles errors in rewriter pipelines. */
export const handleInternalError: httpProxy.ErrorCallback = (
  err,
  _req,
  res
) => {
  logger.error({ error: err }, "Error in http-proxy-middleware pipeline.");
  try {
    writeErrorResponse(res as Response, 500, {
      error: {
        type: "proxy_error",
        message: err.message,
        stack: err.stack,
        proxy_note: `Reverse proxy encountered an error before it could reach the upstream API.`,
      },
    });
  } catch (e) {
    logger.error(
      { error: e },
      `Error writing error response headers, giving up.`
    );
  }
};

const incrementKeyUsage: ProxyResHandlerWithBody = async (_proxyRes, req) => {
  if (QUOTA_ROUTES.includes(req.path)) {
    keyPool.incrementPrompt(req.key?.hash);
    if (req.user) {
      incrementPromptCount(req.user.token);
    }
  }
};

const trackRateLimit: ProxyResHandlerWithBody = async (proxyRes, req) => {
  keyPool.updateRateLimits(req.key!.hash, proxyRes.headers);
};

const copyHttpHeaders: ProxyResHandlerWithBody = async (
  proxyRes,
  _req,
  res
) => {
  Object.keys(proxyRes.headers).forEach((key) => {
    // Omit content-encoding because we will always decode the response body
    if (key === "content-encoding") {
      return;
    }
    // We're usually using res.json() to send the response, which causes express
    // to set content-length. That's not valid for chunked responses and some
    // clients will reject it so we need to omit it.
    if (key === "transfer-encoding") {
      return;
    }
    res.setHeader(key, proxyRes.headers[key] as string);
  });
};


code:
import { config } from "../../../config";
import { logQueue } from "../../../prompt-logging";
import { isCompletionRequest } from "../request";
import { ProxyResHandlerWithBody } from ".";

/** If prompt logging is enabled, enqueues the prompt for logging. */
export const logPrompt: ProxyResHandlerWithBody = async (
  _proxyRes,
  req,
  _res,
  responseBody
) => {
  if (!config.promptLogging) {
    return;
  }
  if (typeof responseBody !== "object") {
    throw new Error("Expected body to be an object");
  }

  // Only log prompts if we're making a request to a completion endpoint
  if (!isCompletionRequest(req)) {
    // Remove this once we're confident that we're not missing any prompts
    req.log.info(
      `Not logging prompt for ${req.path} because it's not a completion endpoint`
    );
    return;
  }

  const model = req.body.model;
  const promptFlattened = flattenMessages(req.body.messages);
  const response = getResponseForModel({ model, body: responseBody });

  logQueue.enqueue({
    model,
    endpoint: req.api,
    promptRaw: JSON.stringify(req.body.messages),
    promptFlattened,
    response,
  });
};

type OaiMessage = {
  role: "user" | "assistant" | "system";
  content: string;
};

const flattenMessages = (messages: OaiMessage[]): string => {
  return messages.map((m) => `${m.role}: ${m.content}`).join("\n");
};

const getResponseForModel = ({
  model,
  body,
}: {
  model: string;
  body: Record<string, any>;
}) => {
  if (model.startsWith("claude")) {
    // TODO: confirm if there is supposed to be a leading space
    return body.completion.trim();
  } else {
    return body.choices[0].message.content;
  }
};


code:
import { Request, Router } from "express";
import * as http from "http";
import { createProxyMiddleware } from "http-proxy-middleware";
import { config } from "../config";
import { logger } from "../logger";
import { createQueueMiddleware } from "./queue";
import { ipLimiter } from "./rate-limit";
import {
  addKey,
  languageFilter,
  checkStreaming,
  finalizeBody,
  limitOutputTokens,
  limitCompletions,
} from "./middleware/request";
import {
  createOnProxyResHandler,
  handleInternalError,
  ProxyResHandlerWithBody,
} from "./middleware/response";

const rewriteRequest = (
  proxyReq: http.ClientRequest,
  req: Request,
  res: http.ServerResponse
) => {
  req.api = "openai";
  const rewriterPipeline = [
    addKey,
    languageFilter,
    checkStreaming,
    limitOutputTokens,
    limitCompletions,
    finalizeBody,
  ];

  try {
    for (const rewriter of rewriterPipeline) {
      rewriter(proxyReq, req, res, {});
    }
  } catch (error) {
    logger.error(error, "Error while executing proxy rewriter");
    proxyReq.destroy(error as Error);
  }
};

const openaiResponseHandler: ProxyResHandlerWithBody = async (
  _proxyRes,
  req,
  res,
  body
) => {
  if (typeof body !== "object") {
    throw new Error("Expected body to be an object");
  }

  if (config.promptLogging) {
    const host = req.get("host");
    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
  }

  res.status(200).json(body);
};

const openaiProxy = createProxyMiddleware({
  target: "https://api.openai.com",
  changeOrigin: true,
  on: {
    proxyReq: rewriteRequest,
    proxyRes: createOnProxyResHandler([openaiResponseHandler]),
    error: handleInternalError,
  },
  selfHandleResponse: true,
  logger,
});
const queuedOpenaiProxy = createQueueMiddleware(openaiProxy);

const openaiRouter = Router();
// Some clients don't include the /v1/ prefix in their requests and users get
// confused when they get a 404. Just fix the route for them so I don't have to
// provide a bunch of different routes for each client's idiosyncrasies.
openaiRouter.use((req, _res, next) => {
  if (!req.path.startsWith("/v1/")) {
    req.url = `/v1${req.url}`;
  }
  next();
});
openaiRouter.get("/v1/models", openaiProxy);
openaiRouter.post("/v1/chat/completions", ipLimiter, queuedOpenaiProxy);
// If a browser tries to visit a route that doesn't exist, redirect to the info
// page to help them find the right URL.
openaiRouter.get("*", (req, res, next) => {
  const isBrowser = req.headers["user-agent"]?.includes("Mozilla");
  if (isBrowser) {
    res.redirect("/");
  } else {
    next();
  }
});
openaiRouter.use((req, res) => {
  logger.warn(`Blocked openai proxy request: ${req.method} ${req.path}`);
  res.status(404).json({ error: "Not found" });
});

export const openai = openaiRouter;


code:
/**
 * Very scuffed request queue. OpenAI's GPT-4 keys have a very strict rate limit
 * of 40000 generated tokens per minute. We don't actually know how many tokens
 * a given key has generated, so our queue will simply retry requests that fail
 * with a non-billing related 429 over and over again until they succeed.
 *
 * Dequeueing can operate in one of two modes:
 * - 'fair': requests are dequeued in the order they were enqueued.
 * - 'random': requests are dequeued randomly, not really a queue at all.
 *
 * When a request to a proxied endpoint is received, we create a closure around
 * the call to http-proxy-middleware and attach it to the request. This allows
 * us to pause the request until we have a key available. Further, if the
 * proxied request encounters a retryable error, we can simply put the request
 * back in the queue and it will be retried later using the same closure.
 */

import type { Handler, Request } from "express";
import { config, DequeueMode } from "../config";
import { keyPool } from "../key-management";
import { logger } from "../logger";
import { AGNAI_DOT_CHAT_IP } from "./rate-limit";

const queue: Request[] = [];
const log = logger.child({ module: "request-queue" });

let dequeueMode: DequeueMode = "fair";

/** Maximum number of queue slots for Agnai.chat requests. */
const AGNAI_CONCURRENCY_LIMIT = 15;
/** Maximum number of queue slots for individual users. */
const USER_CONCURRENCY_LIMIT = 1;

const sameIpPredicate = (incoming: Request) => (queued: Request) =>
  queued.ip === incoming.ip;
const sameUserPredicate = (incoming: Request) => (queued: Request) => {
  const incomingUser = incoming.user ?? { token: incoming.ip };
  const queuedUser = queued.user ?? { token: queued.ip };
  return queuedUser.token === incomingUser.token;
};

export function enqueue(req: Request) {
  let enqueuedRequestCount = 0;
  let isGuest = req.user?.token === undefined;

  if (isGuest) {
    enqueuedRequestCount = queue.filter(sameIpPredicate(req)).length;
  } else {
    enqueuedRequestCount = queue.filter(sameUserPredicate(req)).length;
  }

  // All Agnai.chat requests come from the same IP, so we allow them to have
  // more spots in the queue. Can't make it unlimited because people will
  // intentionally abuse it.
  // Authenticated users always get a single spot in the queue.
  const maxConcurrentQueuedRequests =
    isGuest && req.ip === AGNAI_DOT_CHAT_IP
      ? AGNAI_CONCURRENCY_LIMIT
      : USER_CONCURRENCY_LIMIT;
  if (enqueuedRequestCount >= maxConcurrentQueuedRequests) {
    if (req.ip === AGNAI_DOT_CHAT_IP) {
      // Re-enqueued requests are not counted towards the limit since they
      // already made it through the queue once.
      if (req.retryCount === 0) {
        throw new Error("Too many agnai.chat requests are already queued");
      }
    } else {
      throw new Error("Your IP or token already has a request in the queue");
    }
  }

  queue.push(req);
  req.queueOutTime = 0;

  // shitty hack to remove hpm's event listeners on retried requests
  removeProxyMiddlewareEventListeners(req);

  // If the request opted into streaming, we need to register a heartbeat
  // handler to keep the connection alive while it waits in the queue. We
  // deregister the handler when the request is dequeued.
  if (req.body.stream) {
    const res = req.res!;
    if (!res.headersSent) {
      initStreaming(req);
    }
    req.heartbeatInterval = setInterval(() => {
      if (process.env.NODE_ENV === "production") {
        req.res!.write(": queue heartbeat\n\n");
      } else {
        req.log.info(`Sending heartbeat to request in queue.`);
        const avgWait = Math.round(getEstimatedWaitTime() / 1000);
        const currentDuration = Math.round((Date.now() - req.startTime) / 1000);
        const debugMsg = `queue length: ${queue.length}; elapsed time: ${currentDuration}s; avg wait: ${avgWait}s`;
        req.res!.write(buildFakeSseMessage("heartbeat", debugMsg));
      }
    }, 10000);
  }

  // Register a handler to remove the request from the queue if the connection
  // is aborted or closed before it is dequeued.
  const removeFromQueue = () => {
    req.log.info(`Removing aborted request from queue.`);
    const index = queue.indexOf(req);
    if (index !== -1) {
      queue.splice(index, 1);
    }
    if (req.heartbeatInterval) {
      clearInterval(req.heartbeatInterval);
    }
  };
  req.onAborted = removeFromQueue;
  req.res!.once("close", removeFromQueue);

  if (req.retryCount ?? 0 > 0) {
    req.log.info({ retries: req.retryCount }, `Enqueued request for retry.`);
  } else {
    req.log.info(`Enqueued new request.`);
  }
}

export function dequeue(model: string): Request | undefined {
  // TODO: This should be set by some middleware that checks the request body.
  const modelQueue =
    model === "gpt-4"
      ? queue.filter((req) => req.body.model?.startsWith("gpt-4"))
      : queue.filter((req) => !req.body.model?.startsWith("gpt-4"));

  if (modelQueue.length === 0) {
    return undefined;
  }

  let req: Request;

  if (dequeueMode === "fair") {
    // Dequeue the request that has been waiting the longest
    req = modelQueue.reduce((prev, curr) =>
      prev.startTime < curr.startTime ? prev : curr
    );
  } else {
    // Dequeue a random request
    const index = Math.floor(Math.random() * modelQueue.length);
    req = modelQueue[index];
  }
  queue.splice(queue.indexOf(req), 1);

  if (req.onAborted) {
    req.res!.off("close", req.onAborted);
    req.onAborted = undefined;
  }

  if (req.heartbeatInterval) {
    clearInterval(req.heartbeatInterval);
  }

  // Track the time leaving the queue now, but don't add it to the wait times
  // yet because we don't know if the request will succeed or fail. We track
  // the time now and not after the request succeeds because we don't want to
  // include the model processing time.
  req.queueOutTime = Date.now();
  return req;
}

/**
 * Naive way to keep the queue moving by continuously dequeuing requests. Not
 * ideal because it limits throughput but we probably won't have enough traffic
 * or keys for this to be a problem.  If it does we can dequeue multiple
 * per tick.
 **/
function processQueue() {
  // This isn't completely correct, because a key can service multiple models.
  // Currently if a key is locked out on one model it will also stop servicing
  // the others, because we only track one rate limit per key.
  const gpt4Lockout = keyPool.getLockoutPeriod("gpt-4");
  const turboLockout = keyPool.getLockoutPeriod("gpt-3.5-turbo");

  const reqs: (Request | undefined)[] = [];
  if (gpt4Lockout === 0) {
    reqs.push(dequeue("gpt-4"));
  }
  if (turboLockout === 0) {
    reqs.push(dequeue("gpt-3.5-turbo"));
  }

  reqs.filter(Boolean).forEach((req) => {
    if (req?.proceed) {
      req.log.info({ retries: req.retryCount }, `Dequeuing request.`);
      req.proceed();
    }
  });
  setTimeout(processQueue, 50);
}

/**
 * Kill stalled requests after 5 minutes, and remove tracked wait times after 2
 * minutes.
 **/
function cleanQueue() {
  const now = Date.now();
  const oldRequests = queue.filter(
    (req) => now - (req.startTime ?? now) > 5 * 60 * 1000
  );
  oldRequests.forEach((req) => {
    req.log.info(`Removing request from queue after 5 minutes.`);
    killQueuedRequest(req);
  });

  const index = waitTimes.findIndex(
    (waitTime) => now - waitTime.end > 300 * 1000
  );
  const removed = waitTimes.splice(0, index + 1);
  log.debug(
    { stalledRequests: oldRequests.length, prunedWaitTimes: removed.length },
    `Cleaning up request queue.`
  );
  setTimeout(cleanQueue, 20 * 1000);
}

export function start() {
  processQueue();
  cleanQueue();
  log.info(`Started request queue.`);
}

let waitTimes: { start: number; end: number }[] = [];

/** Adds a successful request to the list of wait times. */
export function trackWaitTime(req: Request) {
  waitTimes.push({
    start: req.startTime!,
    end: req.queueOutTime ?? Date.now(),
  });
}

/** Returns average wait time in milliseconds. */
export function getEstimatedWaitTime() {
  const now = Date.now();
  const recentWaits = waitTimes.filter((wt) => now - wt.end < 300 * 1000);
  if (recentWaits.length === 0) {
    return 0;
  }

  return (
    recentWaits.reduce((sum, wt) => sum + wt.end - wt.start, 0) /
    recentWaits.length
  );
}

export function getQueueLength() {
  return queue.length;
}

export function createQueueMiddleware(proxyMiddleware: Handler): Handler {
  return (req, res, next) => {
    if (config.queueMode === "none") {
      return proxyMiddleware(req, res, next);
    }

    req.proceed = () => {
      proxyMiddleware(req, res, next);
    };

    try {
      enqueue(req);
    } catch (err: any) {
      req.res!.status(429).json({
        type: "proxy_error",
        message: err.message,
        stack: err.stack,
        proxy_note: `Only one request per IP can be queued at a time. If you don't have another request queued, your IP may be in use by another user.`,
      });
    }
  };
}

function killQueuedRequest(req: Request) {
  if (!req.res || req.res.writableEnded) {
    req.log.warn(`Attempted to terminate request that has already ended.`);
    return;
  }
  const res = req.res;
  try {
    const message = `Your request has been terminated by the proxy because it has been in the queue for more than 5 minutes. The queue is currently ${queue.length} requests long.`;
    if (res.headersSent) {
      const fakeErrorEvent = buildFakeSseMessage("proxy queue error", message);
      res.write(fakeErrorEvent);
      res.end();
    } else {
      res.status(500).json({ error: message });
    }
  } catch (e) {
    req.log.error(e, `Error killing stalled request.`);
  }
}

function initStreaming(req: Request) {
  req.log.info(`Initiating streaming for new queued request.`);
  const res = req.res!;
  res.statusCode = 200;
  res.setHeader("Content-Type", "text/event-stream");
  res.setHeader("Cache-Control", "no-cache");
  res.setHeader("Connection", "keep-alive");
  res.setHeader("X-Accel-Buffering", "no"); // nginx-specific fix
  res.flushHeaders();
  res.write("\n");
  res.write(": joining queue\n\n");
}

export function buildFakeSseMessage(type: string, string: string) {
  const fakeEvent = {
    id: "chatcmpl-" + type,
    object: "chat.completion.chunk",
    created: Date.now(),
    model: "",
    choices: [
      {
        delta: { content: `\`\`\`\n[${type}: ${string}]\n\`\`\`\n` },
        index: 0,
        finish_reason: type,
      },
    ],
  };
  return `data: ${JSON.stringify(fakeEvent)}\n\n`;
}

/**
 * http-proxy-middleware attaches a bunch of event listeners to the req and
 * res objects which causes problems with our approach to re-enqueuing failed
 * proxied requests. This function removes those event listeners.
 * We don't have references to the original event listeners, so we have to
 * look through the list and remove HPM's listeners by looking for particular
 * strings in the listener functions. This is an astoundingly shitty way to do
 * this, but it's the best I can come up with.
 */
function removeProxyMiddlewareEventListeners(req: Request) {
  // node_modules/http-proxy-middleware/dist/plugins/default/debug-proxy-errors-plugin.js:29
  // res.listeners('close')
  const RES_ONCLOSE = `Destroying proxyRes in proxyRes close event`;
  // node_modules/http-proxy-middleware/dist/plugins/default/debug-proxy-errors-plugin.js:19
  // res.listeners('error')
  const RES_ONERROR = `Socket error in proxyReq event`;
  // node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js:146
  // req.listeners('aborted')
  const REQ_ONABORTED = `proxyReq.abort()`;
  // node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js:156
  // req.listeners('error')
  const REQ_ONERROR = `if (req.socket.destroyed`;

  const res = req.res!;

  const resOnClose = res
    .listeners("close")
    .find((listener) => listener.toString().includes(RES_ONCLOSE));
  if (resOnClose) {
    res.removeListener("close", resOnClose as any);
  }

  const resOnError = res
    .listeners("error")
    .find((listener) => listener.toString().includes(RES_ONERROR));
  if (resOnError) {
    res.removeListener("error", resOnError as any);
  }

  const reqOnAborted = req
    .listeners("aborted")
    .find((listener) => listener.toString().includes(REQ_ONABORTED));
  if (reqOnAborted) {
    req.removeListener("aborted", reqOnAborted as any);
  }

  const reqOnError = req
    .listeners("error")
    .find((listener) => listener.toString().includes(REQ_ONERROR));
  if (reqOnError) {
    req.removeListener("error", reqOnError as any);
  }
}


code:
import { Request, Response, NextFunction } from "express";
import { config } from "../config";

export const AGNAI_DOT_CHAT_IP = "157.230.249.32";
const RATE_LIMIT_ENABLED = Boolean(config.modelRateLimit);
const RATE_LIMIT = Math.max(1, config.modelRateLimit);
const ONE_MINUTE_MS = 60 * 1000;

const lastAttempts = new Map<string, number[]>();

const expireOldAttempts = (now: number) => (attempt: number) =>
  attempt > now - ONE_MINUTE_MS;

const getTryAgainInMs = (ip: string) => {
  const now = Date.now();
  const attempts = lastAttempts.get(ip) || [];
  const validAttempts = attempts.filter(expireOldAttempts(now));

  if (validAttempts.length >= RATE_LIMIT) {
    return validAttempts[0] - now + ONE_MINUTE_MS;
  } else {
    lastAttempts.set(ip, [...validAttempts, now]);
    return 0;
  }
};

const getStatus = (ip: string) => {
  const now = Date.now();
  const attempts = lastAttempts.get(ip) || [];
  const validAttempts = attempts.filter(expireOldAttempts(now));
  return {
    remaining: Math.max(0, RATE_LIMIT - validAttempts.length),
    reset: validAttempts.length > 0 ? validAttempts[0] + ONE_MINUTE_MS : now,
  };
};

/** Prunes attempts and IPs that are no longer relevant after one minutes. */
const clearOldAttempts = () => {
  const now = Date.now();
  for (const [ip, attempts] of lastAttempts.entries()) {
    const validAttempts = attempts.filter(expireOldAttempts(now));
    if (validAttempts.length === 0) {
      lastAttempts.delete(ip);
    } else {
      lastAttempts.set(ip, validAttempts);
    }
  }
};
setInterval(clearOldAttempts, 10 * 1000);

export const getUniqueIps = () => {
  return lastAttempts.size;
};

export const ipLimiter = (req: Request, res: Response, next: NextFunction) => {
  if (!RATE_LIMIT_ENABLED) {
    next();
    return;
  }

  // Exempt Agnai.chat from rate limiting since it's shared between a lot of
  // users. Dunno how to prevent this from being abused without some sort of
  // identifier sent from Agnaistic to identify specific users.
  if (req.ip === AGNAI_DOT_CHAT_IP) {
    next();
    return;
  }

  // If user is authenticated, key rate limiting by their token. Otherwise, key
  // rate limiting by their IP address. Mitigates key sharing.
  const rateLimitKey = req.user?.token || req.ip;

  const { remaining, reset } = getStatus(rateLimitKey);
  res.set("X-RateLimit-Limit", config.modelRateLimit.toString());
  res.set("X-RateLimit-Remaining", remaining.toString());
  res.set("X-RateLimit-Reset", reset.toString());

  const tryAgainInMs = getTryAgainInMs(rateLimitKey);
  if (tryAgainInMs > 0) {
    res.set("Retry-After", tryAgainInMs.toString());
    res.status(429).json({
      error: {
        type: "proxy_rate_limited",
        message: `This proxy is rate limited to ${
          config.modelRateLimit
        } model requests per minute. Please try again in ${Math.ceil(
          tryAgainInMs / 1000
        )} seconds.`,
      },
    });
  } else {
    next();
  }
};


code:
/* Accepts incoming requests at either the /kobold or /openai routes and then
routes them to the appropriate handler to be forwarded to the OpenAI API.
Incoming OpenAI requests are more or less 1:1 with the OpenAI API, but only a
subset of the API is supported. Kobold requests must be transformed into
equivalent OpenAI requests. */

import * as express from "express";
import { gatekeeper } from "./auth/gatekeeper";
import { kobold } from "./kobold";
import { openai } from "./openai";

const router = express.Router();

router.use(gatekeeper);
router.use("/kobold", kobold);
router.use("/openai", openai);

// Each client handles the endpoints input by the user in slightly different
// ways, eg TavernAI ignores everything after the hostname in Kobold mode
function rewriteTavernRequests(
  req: express.Request,
  _res: express.Response,
  next: express.NextFunction
) {
  // Requests coming into /api/v1 are actually requests to /proxy/kobold/api/v1
  if (req.path.startsWith("/api/v1")) {
    req.url = req.url.replace("/api/v1", "/proxy/kobold/api/v1");
  }
  next();
}

export { rewriteTavernRequests };
export { router as proxyRouter };


code:
import { assertConfigIsValid, config } from "./config";
import "source-map-support/register";
import express from "express";
import cors from "cors";
import pinoHttp from "pino-http";
import childProcess from "child_process";
import { logger } from "./logger";
import { keyPool } from "./key-management";
import { adminRouter } from "./admin/routes";
import { proxyRouter, rewriteTavernRequests } from "./proxy/routes";
import { handleInfoPage } from "./info-page";
import { logQueue } from "./prompt-logging";
import { start as startRequestQueue } from "./proxy/queue";
import { init as initUserStore } from "./proxy/auth/user-store";
import { checkOrigin } from "./proxy/check-origin";

const PORT = config.port;

const app = express();
// middleware
app.use("/", rewriteTavernRequests);
app.use(
  pinoHttp({
    quietReqLogger: true,
    logger,
    autoLogging: {
      ignore: (req) => {
        const ignored = ["/proxy/kobold/api/v1/model", "/health"];
        return ignored.includes(req.url as string);
      },
    },
    redact: {
      paths: [
        "req.headers.cookie",
        'res.headers["set-cookie"]',
        "req.headers.authorization",
        'req.headers["x-forwarded-for"]',
        'req.headers["x-real-ip"]',
        'req.headers["true-client-ip"]',
        'req.headers["cf-connecting-ip"]',
      ],
      censor: "********",
    },
  })
);

app.get("/health", (_req, res) => res.sendStatus(200));
app.use((req, _res, next) => {
  req.startTime = Date.now();
  req.retryCount = 0;
  next();
});
app.use(cors());
app.use(
  express.json({ limit: "10mb" }),
  express.urlencoded({ extended: true, limit: "10mb" })
);

// TODO: Detect (or support manual configuration of) whether the app is behind
// a load balancer/reverse proxy, which is necessary to determine request IP
// addresses correctly.
app.set("trust proxy", true);

// routes
app.use(checkOrigin);
app.get("/", handleInfoPage);
app.use("/admin", adminRouter);
app.use("/proxy", proxyRouter);

// 500 and 404
app.use((err: any, _req: unknown, res: express.Response, _next: unknown) => {
  if (err.status) {
    res.status(err.status).json({ error: err.message });
  } else {
    logger.error(err);
    res.status(500).json({
      error: {
        type: "proxy_error",
        message: err.message,
        stack: err.stack,
        proxy_note: `Reverse proxy encountered an internal server error.`,
      },
    });
  }
});
app.use((_req: unknown, res: express.Response) => {
  res.status(404).json({ error: "Not found" });
});

async function start() {
  logger.info("Server starting up...");
  await setBuildInfo();

  logger.info("Checking configs and external dependencies...");
  await assertConfigIsValid();

  keyPool.init();

  if (config.gatekeeper === "user_token") {
    await initUserStore();
  }

  if (config.promptLogging) {
    logger.info("Starting prompt logging...");
    logQueue.start();
  }

  if (config.queueMode !== "none") {
    logger.info("Starting request queue...");
    startRequestQueue();
  }

  app.listen(PORT, async () => {
    logger.info({ port: PORT }, "Now listening for connections.");
    registerUncaughtExceptionHandler();
  });

  logger.info(
    { build: process.env.BUILD_INFO, nodeEnv: process.env.NODE_ENV },
    "Startup complete."
  );
}

function registerUncaughtExceptionHandler() {
  process.on("uncaughtException", (err: any) => {
    logger.error(
      { err, stack: err?.stack },
      "UNCAUGHT EXCEPTION. Please report this error trace."
    );
  });
  process.on("unhandledRejection", (err: any) => {
    logger.error(
      { err, stack: err?.stack },
      "UNCAUGHT PROMISE REJECTION. Please report this error trace."
    );
  });
}

/**
 * Attepts to collect information about the current build from either the
 * environment or the git repo used to build the image (only works if not
 * .dockerignore'd). If you're running a sekrit club fork, you can no-op this
 * function and set the BUILD_INFO env var manually, though I would prefer you
 * didn't set it to something misleading.
 */
async function setBuildInfo() {
  // Render .dockerignore's the .git directory but provides info in the env
  if (process.env.RENDER) {
    const sha = process.env.RENDER_GIT_COMMIT?.slice(0, 7) || "unknown SHA";
    const branch = process.env.RENDER_GIT_BRANCH || "unknown branch";
    const repo = process.env.RENDER_GIT_REPO_SLUG || "unknown repo";
    const buildInfo = `${sha} (${branch}@${repo})`;
    process.env.BUILD_INFO = buildInfo;
    logger.info({ build: buildInfo }, "Got build info from Render config.");
    return;
  }

  try {
    // Ignore git's complaints about dubious directory ownership on Huggingface
    // (which evidently runs dockerized Spaces on Windows with weird NTFS perms)
    if (process.env.SPACE_ID) {
      childProcess.execSync("git config --global --add safe.directory /app");
    }

    const promisifyExec = (cmd: string) =>
      new Promise((resolve, reject) => {
        childProcess.exec(cmd, (err, stdout) =>
          err ? reject(err) : resolve(stdout)
        );
      });

    const promises = [
      promisifyExec("git rev-parse --short HEAD"),
      promisifyExec("git rev-parse --abbrev-ref HEAD"),
      promisifyExec("git config --get remote.origin.url"),
      promisifyExec("git status --porcelain"),
    ].map((p) => p.then((result: any) => result.toString().trim()));

    let [sha, branch, remote, status] = await Promise.all(promises);

    remote = remote.match(/.*[\/:]([\w-]+)\/([\w\-\.]+?)(?:\.git)?$/) || [];
    const repo = remote.slice(-2).join("/");
    status = status
      // ignore Dockerfile changes since that's how the user deploys the app
      .split("\n")
      .filter((line: string) => !line.endsWith("Dockerfile") && line);

    const changes = status.length > 0;

    const build = `${sha}${changes ? " (modified)" : ""} (${branch}@${repo})`;
    process.env.BUILD_INFO = build;
    logger.info({ build, status, changes }, "Got build info from Git.");
  } catch (error: any) {
    logger.error(
      {
        error,
        stdout: error.stdout.toString(),
        stderr: error.stderr.toString(),
      },
      "Failed to get commit SHA.",
      error
    );
    process.env.BUILD_INFO = "unknown";
  }
}

start();


code:
import { Express } from "express-serve-static-core";
import { Key } from "../key-management/key-pool";
import { User } from "../proxy/auth/user-store";

declare global {
  namespace Express {
    interface Request {
      key?: Key;
      api: "kobold" | "openai" | "anthropic";
      user?: User;
      isStreaming?: boolean;
      startTime: number;
      retryCount: number;
      queueOutTime?: number;
      onAborted?: () => void;
      proceed: () => void;
      heartbeatInterval?: NodeJS.Timeout;
    }
  }
}


module: CommonJS

moduleResolution: node

outDir: build


code:
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
*.swp
*.swo

# C extensions
*.so

# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# IPython Notebook
.ipynb_checkpoints

# pyenv
.python-version

# celery beat schedule file
celerybeat-schedule

# dotenv
.env

# virtualenv
venv/
ENV/

# Spyder project settings
.spyderproject

# Rope project settings
.ropeproject

# Start.tw
src/config/start.tw

path: .gitignore

license: mit


code:
further development:
-specialized slave schools
-fortifications
-more levels for militia edict (further militarize society)
-conquering other arcologies?

Events:
-famous criminal escapes to the arcology, followed by another arcology police force

Bugs:
-sometimes troop counts breaks
-sometimes rebel numbers have fractionary parts


code:
# How to vector art

Please read the whole document before starting to work.
Some aspects are not obvious right away.

Note: This does not actually describe how to be an artist.

## TL;DR

    killall inkscape
    artTools/vector_layer_split.py artTools/vector_source.svg tw src/art/vector/layers/
    compile

## 1. Be an artist

Make changes to the vector_source.svg.
Inkscape was thoroughly tested.
Adobe Illustrator might work decently, too.

## 2. Respect the structure

While editing, keep the Layers in mind.

* In Inkscape, Layers are special "Inkscape groups".
* In Illustrator, Layers are groups with user-definded IDs.
* All Layers should have an ID that is globally unique
  (not just within their subtree).

* Please use anonymous groups only for the ease of editing. Remove all "helper" groups before finally saving the file.
* Anonymous groups can be used for continous scaling (of e.g. boobs).

* Every asset that should go into a separate file needs to be in a labelled group
  (even if that is a group with only one shape).
* There are some globally available styles defined as CSS classes (e.g. skin, hair).
  Use them if your asset should be changed upon display.
  Do not set the style directly in the object.

## 3. Fix the document (before commiting, Inkscape only)

Inkscape shows weird behaviour in some regards.
If you use Inkscape, close the document and run

    python3 inkscape_svg_fixup.py vector_source.svg

before continuing. Open the file in Inkscape and save it again.
You need to make a minor change, as Inkscape will not save a unchanged file
(move the notes back and forth or something). The fixup does not produce
the same linebreaks as Inkscape or Illustrator and git will go mad because
the whole seems to have changed.

What it does:
* Adobe Illustrator uses group IDs as layer labels.
  Inkscape however uses layer labels and a separate, auto-generated group ID.
  inkscape_svg_fixup.py overwrites the group ID with the Inkscape layer label
  so they are synchronised with Inkscape layer labels.
* Inkscape copies the global style definition into the object *every time*
  the object is edited. If an object references a CSS class AND at the same time
  has a local style, the local style is removed
  so global dynamic styling is possible later on.

Note: Behaviour of Adobe Illustrator is untested.

## 4. Split the layers

Execute

    python3 vector_layer_split.py vector_source.svg tw ../src/art/vector/layers/

For revamped art

	python3 vector_revamp_layer_split.py vector_revamp_source.svg tw ../src/art/vector_revamp/layers/

. This application reads all groups in `vector_source.svg`.
Each group is stored in a separate file in the target directory `/src/art/vector/layers/`.
The group ID sets the file name. Therefore, the group ID **must not** contain spaces or any other weird characters.

Also consider:
* A group with ID ending in _ is ignored. Use Layers ending in _ to keep overview.
* A group with ID starting with "g" (Inkscape) or "XMLID" (Illustrator) is also ignored.
* Existing files are overwritten **without enquiry**.
* The target directory is not emptied. If a file is no longer needed, you should remove it manually.
* This procedure removes global definitions. This means, SVG filters are currently not supported.

Available output formats are `svg` and `tw`.
`svg` output exists for debug reasons.
`tw` embeds the SVG data into Twine files, but removes the global style definitions so they can be set during display.

## 5. Edit the code

`/src/art/` contains Twine code which shows the assets in the story.
There are many helpful comments in `/src/art/artWidgets.tw`.
The code also generates the previously removed global style definitions on the fly and per display.

## 6. Compile the story

Use the provided compile script (Windows batch or shell) for compiling.

## 7. Advanced usage

You can define multiple CSS classes to one element, e.g. "skin torso". When procedurally generating CSS, you can then set a global "skin" style, and a different one for "torso".

You can put variable text into the image using single quotes, e.g. "'+_tattooText+'". The single quotes *break* the quoting in Twine so the content of the Twine variable `_tattooText` is shown upon display. You can even align text on paths.

An anonymous group can be used for dynamic transformation. However, finding the correct parameters for the affine transformation matrix can be cumbersome. Use any method you see fit. See `src/art/vector/Boob.tw` for one example of the result.


code:
#!/usr/bin/env python3

'''
Application for "fixing" SVGs edited with Inkscape

These problems are addressed:

* Inkscape notoriously copies class styles into the object definitions.
https://bugs.launchpad.net/inkscape/+bug/167937

* Inkscape uses labels on layers. Layers are basically named groups.
  Inkscape does not sync the group id with the layer label.

Usage Example:
python3 inkscape_svg_fixup.py vector_source.svg

Note:
The output of lxml differs greatly from standard SVG indentation.
Please open and save the file in Inkscape before committing.
'''

import lxml.etree as etree
import sys

def fix(tree):
  # know namespaces
  ns = {
    'svg' : 'http://www.w3.org/2000/svg',
    'inkscape' : 'http://www.inkscape.org/namespaces/inkscape'
  }

  # find document global style definition
  # mangle it and interpret as python dictionary
  style_element = tree.find('./svg:style',namespaces=ns)
  style_definitions = style_element.text
  pythonic_style_definitions = '{'+style_definitions.\
  replace('\t.','"').replace('{','":"').replace('}','",').\
  replace('/*','#')+'}'
  styles = eval(pythonic_style_definitions)

  # go through all SVG elements
  for elem in tree.iter():
    if (elem.tag == etree.QName(ns['svg'], 'g')):
      # compare inkscape label with group element ID
      l = elem.get(etree.QName(ns['inkscape'], 'label'))
      if l:
        i = elem.get('id')
        if (i != l):
          print("Overwriting ID %s with Label %s..."%(i, l))
          elem.set('id', l)

    # remove all offending style attributes
    s = elem.get('style')
    c = elem.get('class')
    if (c and s):
      s = s.lower()
      c = c.split(' ')[0] # regard main style only
      cs = ''
      if c in styles:
        cs = styles[c].strip('; ').lower()
      if (c not in styles):
        print("Object id %s references unknown style class %s."%(i,c))
      else:
        if (cs != s.strip('; ')):
          print("Style %s removed from object id %s differed from class %s style %s."%(s,i,c,cs))
        del elem.attrib["style"]

if __name__ == "__main__":
  input_file = sys.argv[1]
  tree = etree.parse(input_file)
  fix(tree)
  # store SVG into file (input file is overwritten)
  svg = etree.tostring(tree, pretty_print=True)
  with open(input_file, 'wb') as f:
    f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
    f.write(svg)


code:
#!/usr/bin/env python3

'''
Application for procedural content adaption
*THIS IS VERY EXPERIMENTAL*
Contains a very poor man's implementation of spline mesh warping.

This application parses SVG path data of an outfit sample aligned to a body part.
The outfit is then replicated for other shapes of the same body part.

Example data is geared towards generating a strap outfit for boobs and torso
for all sizes of boobs and all shapes of torsos based on a single outfit for
boobs of size 2 and a hourglass torso respectively.

Limitations:
* handles paths only
* only svg.path Line and CubicBezier are tested
* only the aforementioned examples are tested

Usage:
python3 vector_clothing_replicator.py infile clothing bodypart destinationfile

Usage Example:
python3 vector_clothing_replicator.py vector_source.svg Straps Boob vector_destination.svg
python3 vector_clothing_replicator.py vector_source.svg Straps Torso vector_destination.svg
'''

from svg.path import parse_path
import copy
import lxml.etree as etree
import sys

REFERENCE_PATH_SAMPLES = 200
EMBED_REPLICATIONS = True # wether to embed all replications into the input file or output separate files

input_file = sys.argv[1]
clothing = sys.argv[2]
bodypart = sys.argv[3]
output_file_embed = sys.argv[4]

# TODO: make these configurable
output_file_pattern = '%s_%s_%s.svg' #bodypart, target_id, clothing

if ('Torso' == bodypart):
  xpath_shape = './svg:g[@id="Torso_"]/svg:g[@id="Torso_%s"]/svg:path[@class="skin torso"]/@d' # TODO: formulate more general, independent of style
  xpath_outfit_container = '//svg:g[@id="Torso_Outfit_%s_"]'%(clothing)
  xpath_outfit = '//svg:g[@id="Torso_Outfit_%s_%s"]'%(clothing,'%s')
  target_ids = "Unnatural,Hourglass,Normal".split(",")
  reference_id = "Hourglass"
else:
  raise RuntimeError("Please specify a bodypart for clothing to replicate.")

tree = etree.parse(input_file)
ns = {'svg' : 'http://www.w3.org/2000/svg'}

canvas = copy.deepcopy(tree)
for e in canvas.xpath('./svg:g',namespaces=ns)+canvas.xpath('./svg:path',namespaces=ns):
  # TODO: this should be "remove all objects, preserve document properties"
  e.getparent().remove(e)

def get_points(xpath_shape):
  '''
  This funciton extracts reference paths by the given xpath selector.
  Each path is used to sample a fixed number of points.
  '''
  paths_data = tree.xpath(xpath_shape,namespaces=ns)
  points = []
  path_length = None
  for path_data in paths_data:
    p = parse_path(path_data)
    points += [
      p.point(1.0/float(REFERENCE_PATH_SAMPLES)*i)
      for i in range(REFERENCE_PATH_SAMPLES)
    ]
  if (not points):
    raise RuntimeError(
      'No paths for reference points found by selector "%s".'%(xpath_shape)
    )
  return points

def point_movement(point, reference_points, target_points):
  '''
  For a given point, finds the nearest point in the reference path.
  Gives distance vector from the nearest reference point to the
  respective target reference point.
  '''
  distances = [abs(point-reference_point) for reference_point in reference_points]
  min_ref_dist_idx = min(enumerate(distances), key=lambda x:x[1])[0]
  movement = target_points[min_ref_dist_idx] - reference_points[min_ref_dist_idx]
  return movement

reference_points = get_points(xpath_shape%(reference_id))
container = tree.xpath(xpath_outfit_container,namespaces=ns)
if (len(container) != 1):
  raise RuntimeError('Outfit container selector "%s" does not yield exactly one layer.'%(xpath_outfit_container))
container = container[0]
outfit_source = container.xpath(xpath_outfit%(reference_id),namespaces=ns)
if (len(outfit_source) != 1):
  raise RuntimeError('Outfit source selector "%s" does not yield exactly one outfit layer in container selected by "%s".'%(xpath_outfit%(reference_id), xpath_outfit_container))
outfit_source = outfit_source[0]

for target_id in target_ids:
  print(
    'Generating variant "%s" of clothing "%s" for bodypart "%s"...'%
    (target_id, clothing, bodypart)
  )
  outfit = copy.deepcopy(outfit_source)
  paths = outfit.xpath('./svg:path',namespaces=ns)
  if target_id == reference_id:
    print("This is the source variant. Skipping...")
  else:
    layerid = outfit.get('id').replace('_%s'%(reference_id),'_%s'%(target_id))
    outfit.set('id', layerid)
    outfit.set(etree.QName('http://www.inkscape.org/namespaces/inkscape', 'label'), layerid) # for the Inkscape-users
    target_points = get_points(xpath_shape%(target_id))
    if (len(reference_points) != len(target_points)):
      raise RuntimeError(
        ('Different amounts of sampled points in reference "%s" and target "%s" paths. '+
        'Selector "%s" probably matches different number of paths in the two layers.')%
        (reference_id, target_id, xpath_shape)
      )
    for path in paths:
      path_data = path.get("d")
      p = parse_path(path_data)
      for segment in p:
        original_distance = abs(segment.end-segment.start)
        start_movement = point_movement(segment.start, reference_points, target_points)
        segment.start += start_movement
        end_movement = point_movement(segment.end, reference_points, target_points)
        segment.end += end_movement
        distance = abs(segment.end-segment.start)
        try:
          # enhance position of CubicBezier control points
          # amplification is relative to the distance gained by movement
          segment.control1 += start_movement
          segment.control1 += (segment.control1-segment.start)*(distance/original_distance-1.0)
          segment.control2 += end_movement
          segment.control2 += (segment.control2-segment.end)*(distance/original_distance-1.0)
        except AttributeError as ae:
          # segment is not a CubicBezier
          pass
      path.set("d", p.d())
    if EMBED_REPLICATIONS:
      container.append(outfit)
  if not EMBED_REPLICATIONS:
    container = copy.deepcopy(canvas).xpath('.',namespaces=ns)[0]
    container.append(outfit)

  if not EMBED_REPLICATIONS:
    svg = etree.tostring(container, pretty_print=True)
    with open((output_file_pattern%(bodypart, target_id, clothing)).lower(), 'wb') as f:
      f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
      f.write(svg)

if EMBED_REPLICATIONS:
  svg = etree.tostring(tree, pretty_print=True)
  with open(output_file_embed, 'wb') as f:
    f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
    f.write(svg)


code:
#!/usr/bin/env python3

'''
Application for splitting groups from one SVG file into separate files

Usage:
python3 vector_layer_split.py infile format outdir

Usage Example:
python3 vector_layer_split.py vector_source.svg tw ../src/art/vector/layers/
'''

import lxml.etree as etree
import sys
import os
import copy
import re
import inkscape_svg_fixup

input_file = sys.argv[1]
output_format = sys.argv[2]
output_directory = sys.argv[3]
if not os.path.exists(output_directory):
    os.makedirs(output_directory)

ns = {
  'svg' : 'http://www.w3.org/2000/svg',
  'inkscape' : 'http://www.inkscape.org/namespaces/inkscape',
  'sodipodi':"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd",
}

tree = etree.parse(input_file)
inkscape_svg_fixup.fix(tree)

# prepare output template
template = copy.deepcopy(tree)
root = template.getroot()

# remove all svg root attributes except document size
for a in root.attrib:
  if (a != "viewBox"):
    del root.attrib[a]

# add placeholder for CSS class (needed for workaround for non HTML 5.1 compliant browser)
if output_format == 'tw':
  root.attrib["class"] = "'+_art_display_class+'"

# remove all content, including metadata
# for twine output, style definitions are removed, too
defs = None
for e in root:
  if (e.tag == etree.QName(ns['svg'], 'defs')):
    defs = e
  if (e.tag == etree.QName(ns['svg'], 'g') or
      e.tag == etree.QName(ns['svg'], 'metadata') or
      e.tag == etree.QName(ns['svg'], 'defs') or
      e.tag == etree.QName(ns['sodipodi'], 'namedview') or
      (output_format == 'tw' and e.tag == etree.QName(ns['svg'], 'style'))
      ):
    root.remove(e)
# template preparation finished

# prepare regex for later use
regex_xmlns = re.compile(' xmlns[^ ]+',)
regex_space = re.compile('[>][ ]+[<]',)

# find all groups
layers = tree.xpath('//svg:g',namespaces=ns)
for layer in layers:
  i = layer.get('id')
  if ( # disregard non-content groups
    i.endswith("_") or # manually suppressed with underscore
    i.startswith("XMLID") or # Illustrator generated group
    i.startswith("g") # Inkscape generated group
  ):
    continue
  # create new canvas
  output = copy.deepcopy(template)
  # copy all shapes into template
  canvas = output.getroot()
  for e in layer:
    canvas.append(e)
  # represent template as SVG (binary string)
  svg = etree.tostring(output, pretty_print=False)
  # poor man's conditional defs insertion
  # TODO: extract only referenced defs (filters, gradients, ...)
  # TODO: detect necessity by traversing the elements. do not stupidly search in the string representation
  if ("filter:" in svg.decode('utf-8')):
    # it seems there is a filter referenced in the generated SVG, re-insert defs from main document
    canvas.insert(0,defs)
    # re-generate output
    svg = etree.tostring(output, pretty_print=False)

  if (output_format == 'tw'):
    # remove unnecessary attributes
    # TODO: never generate unnecessary attributes in the first place
    svg = svg.decode('utf-8')
    svg = regex_xmlns.sub('',svg)
    svg = svg.replace(' inkscape:connector-curvature="0"','') # this just saves space
    svg = svg.replace('\n','').replace('\r','') # print cannot be multi-line
    svg = regex_space.sub('><',svg) # remove indentaion
    svg = svg.replace('svg:','') # svg namespace was removed
    svg = svg.replace('<g ','<g transform="\'+_art_transform+\'"') # internal groups are used for scaling
    svg = svg.encode('utf-8')

  # save SVG string to file
  i = layer.get('id')
  output_path = os.path.join(output_directory,i+'.'+output_format)
  with open(output_path, 'wb') as f:
    if (output_format == 'svg'):
      # Header for normal SVG (XML)
      f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
      f.write(svg)
    elif (output_format == 'tw'):
      # Header for SVG in Twine file (SugarCube print statement)
      f.write((':: Art_Vector_%s [nobr]\n\n'%(i)).encode("utf-8"))
      f.write("<<print '<html>".encode("utf-8"))
      f.write(svg)
      f.write("</html>' >>".encode("utf-8"))


code:
#!/usr/bin/env python3

'''
Application for splitting groups from one SVG file into separate files

Usage:
python3 vector_layer_split.py infile format outdir

Usage Example:
python3 vector_layer_split.py vector_source.svg tw ../src/art/vector/layers/
'''

import lxml.etree as etree
from lxml.etree import XMLParser, parse

import sys
import os
import copy
import re
import inkscape_svg_fixup

input_file = sys.argv[1]
output_format = sys.argv[2]
output_directory = sys.argv[3]

if not os.path.exists(output_directory):
    os.makedirs(output_directory)

ns = {
    'svg': 'http://www.w3.org/2000/svg',
    'inkscape': 'http://www.inkscape.org/namespaces/inkscape',
    'sodipodi': "http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd",
}

p = XMLParser(huge_tree=True)
tree = parse(input_file, parser=p)
#tree = etree.parse(input_file)
inkscape_svg_fixup.fix(tree)

# prepare output template
template = copy.deepcopy(tree)
root = template.getroot()

# remove all svg root attributes except document size
for a in root.attrib:
    if (a != "viewBox"):
        del root.attrib[a]

# add placeholder for CSS class (needed for workaround for non HTML 5.1 compliant browser)
if output_format == 'tw':
    root.attrib["class"] = "'+_art_display_class+'"

# remove all content, including metadata
# for twine output, style definitions are removed, too
defs = None
for e in root:
    if (e.tag == etree.QName(ns['svg'], 'defs')):
        defs = e
    if (e.tag == etree.QName(ns['svg'], 'g') or
            e.tag == etree.QName(ns['svg'], 'metadata') or
            e.tag == etree.QName(ns['svg'], 'defs') or
            e.tag == etree.QName(ns['sodipodi'], 'namedview') or
            (output_format == 'tw' and e.tag == etree.QName(ns['svg'], 'style'))
    ):
        root.remove(e)
# template preparation finished

# prepare regex for later use
regex_xmlns = re.compile(' xmlns[^ ]+', )
regex_space = re.compile('[>][ ]+[<]', )
regex_transformValue = re.compile('(?<=transformVariableName=")[^"]*', )
regex_transformVar = re.compile('transformVariableName="[^"]*"', )
regex_transformAttr = re.compile('transform="[^"]*"', )
# find all groups
layers = tree.xpath('//svg:g', namespaces=ns)
for layer in layers:
    i = layer.get('id')
    if (  # disregard non-content groups
            i.endswith("_") or  # manually suppressed with underscore
            i.startswith("XMLID") or  # Illustrator generated group
            i.startswith("g")  # Inkscape generated group
    ):
        continue
    # create new canvas
    output = copy.deepcopy(template)
    # copy all shapes into template
    canvas = output.getroot()
    for e in layer:
        canvas.append(e)
    # represent template as SVG (binary string)
    svg = etree.tostring(output, pretty_print=False)
    # poor man's conditional defs insertion
    # TODO: extract only referenced defs (filters, gradients, ...)
    # TODO: detect necessity by traversing the elements. do not stupidly search in the string representation
    if ("filter:" in svg.decode('utf-8')):
        # it seems there is a filter referenced in the generated SVG, re-insert defs from main document
        canvas.insert(0, defs)
        # re-generate output
        svg = etree.tostring(output, pretty_print=False)
    elif ("clip-path=" in svg.decode('utf-8')):
        # it seems there is a clip path referenced in the generated SVG, re-insert defs from main document
        canvas.insert(0, defs)
        # re-generate output
        svg = etree.tostring(output, pretty_print=False)

    if (output_format == 'tw'):
        # remove unnecessary attributes
        # TODO: never generate unnecessary attributes in the first place
        svg = svg.decode('utf-8')
        svg = regex_xmlns.sub('', svg)
        svg = svg.replace(' inkscape:connector-curvature="0"', '')  # this just saves space
        svg = svg.replace('\n', '').replace('\r', '')  # print cannot be multi-line
        svg = regex_space.sub('><', svg)  # remove indentaion
        svg = svg.replace('svg:', '')  # svg namespace was removed

        transformGroups = regex_transformVar.findall(svg)

        if (len(transformGroups) > 0):
            svg = regex_transformAttr.sub('', svg)
            for transformGroup in transformGroups:
                transformValue = regex_transformValue.search(transformGroup)
                if (transformValue is not None):
                    svg = svg.replace(transformGroup, ' transform="\'+' + transformValue.group() + '+\'" ')  # internal groups are used for scaling

        svg = svg.encode('utf-8')

    # save SVG string to file
    i = layer.get('id')
    output_path = os.path.join(output_directory, i + '.' + output_format)
    with open(output_path, 'wb') as f:
        if (output_format == 'svg'):
            # Header for normal SVG (XML)
            f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
            f.write(svg)
        elif (output_format == 'tw'):
            # Header for SVG in Twine file (SugarCube print statement)
            f.write((':: Art_Vector_Revamp_%s [nobr]\n\n' % (i)).encode("utf-8"))
            f.write("<<print '<html>".encode("utf-8"))
            f.write(svg)
            f.write("</html>' >>".encode("utf-8"))


code:
# Ignore everything in this directory
*
# Except the following:
!.gitignore
!resources/vector/*


code:
#!/bin/bash

while [[ "$1" ]]
do
	case $1 in
	--insane)
		insane="true"
	;;
	*)
		echo "Unknown argument $1"
		exit 1
	esac
	shift
done

# Find and insert current commit
COMMIT=$(git rev-parse --short HEAD)
sed -Ei "s/build .releaseID/\0 commit $COMMIT/" src/gui/mainMenu/AlphaDisclaimer.tw

if [[ ! "$insane" ]]
then
	# Run sanity check.
	./sanityCheck
fi

ARCH="$(uname -m)"
if [ "$ARCH" = "x86_64" ]
then
	echo "x64 arch"
	./devTools/tweeGo/tweego_nix64 -o bin/FC_pregmod.html src/ || build_failed="true"
elif echo "$ARCH" | grep -Ee '86$' > /dev/null
then
	echo "x86 arch"
	./devTools/tweeGo/tweego_nix86 -o bin/FC_pregmod.html src/ || build_failed="true"
elif echo "$ARCH" | grep -Ee '^arm' > /dev/null
then
	echo "arm arch"
	# tweego doesn't provide arm binaries so you have to build it yourself
	export TWEEGO_PATH=devTools/tweeGo/storyFormats
	tweego -o bin/FC_pregmod.html src/ || build_failed="true"
else
	exit 2
fi

# Revert AlphaDisclaimer for next compilation
git checkout -- src/gui/mainMenu/AlphaDisclaimer.tw

if [ "$build_failed" = "true" ]
then
	exit 1
fi

#Make the output prettier, replacing \t with a tab and \n with a newline
sed -i -e '/^.*<div id="store-area".*$/s/\\t/\t/g' -e '/^.*<div id="store-area".*$/s/\\n/\n/g'  bin/FC_pregmod.html

path: compile


code:
#!/bin/bash

# Run sanity check.
./sanityCheck

HASH="$(git rev-list -n 1 --abbrev-commit HEAD)"
ARCH="$(uname -m)"
if [ "$ARCH" = "x86_64" ]
then
	echo "x64 arch"
	./devTools/tweeGo/tweego_nix64 -o "bin/FC_pregmod_$HASH.html" src/
elif echo "$ARCH" | grep -Ee '86$' > /dev/null
then
	echo "x86 arch"
	./devTools/tweeGo/tweego_nix86 -o "bin/FC_pregmod_$HASH.html" src/
elif echo "$ARCH" | grep -Ee '^arm' > /dev/null
then
	echo "arm arch"
	# tweego doesn't provide arm binaries so you have to build it yourself
	export TWEEGO_PATH=devTools/tweeGo/storyFormats
	tweego -o "bin/FC_pregmod_$HASH.html" src/
else
	exit 2
fi

#Make the output prettier, replacing \t with a tab and \n with a newline
sed -i -e '/^<div id="store-area".*$/s/\\t/\t/g' -e '/^<div id="store-area".*$/s/\\n/\n/g'  "bin/FC_pregmod_$HASH.html"

echo "FC_pregmod_$HASH.html" compilation finished.

path: compile-git


code:
@echo off
:: Free Cities Basic Compiler - Windows

:: Set working directory
pushd %~dp0

:: Run the appropriate compiler for the user's CPU architecture.
if %PROCESSOR_ARCHITECTURE% == AMD64 (
    CALL "%~dp0devTools\tweeGo\tweego_win64.exe" -o "%~dp0bin/FC_pregmod.html" "%~dp0src"
) else (
    CALL "%~dp0devTools\tweeGo\tweego_win86.exe" -o "%~dp0bin/FC_pregmod.html" "%~dp0src"
)

popd
ECHO Done


code:
@echo off
:: Free Cities Basic Compiler - Windows

:: Set working directory
pushd %~dp0

:: See if we can find a git installation
setlocal enabledelayedexpansion

for %%k in (HKCU HKLM) do (
    for %%w in (\ \Wow6432Node\) do (
        for /f "skip=2 delims=: tokens=1*" %%a in ('reg query "%%k\SOFTWARE%%wMicrosoft\Windows\CurrentVersion\Uninstall\Git_is1" /v InstallLocation 2^> nul') do (
            for /f "tokens=3" %%z in ("%%a") do (
                set GIT=%%z:%%b
				set GITFOUND=yes
                goto FOUND
            )
        )
    )
)
:FOUND
if %GITFOUND% == yes (
	set "PATH=%GIT%bin;%PATH%"
	bash --login -c ./sanityCheck
)

:: Compile the game
call "%~dp0compile.bat"

if %GITFOUND% == yes (
	:: Make the output prettier, replacing \t with a tab and \n with a newline
	bash -c "sed -i -e '/^.*<div id=\"store-area\".*$/s/\\\t/\t/g' -e '/^.*<div id=\"store-area\".*$/s/\\\n/\n/g' bin/FC_pregmod.html"

	:: Revert ./src/init/storyInit.tw for next compilation
	git checkout -- ./src/init/storyInit.tw
)

popd
PAUSE


code:
@echo off
:: Free Cities Basic Compiler - Windows

:: Set working directory
pushd %~dp0

:: Compile the game
call "%~dp0compile.bat"

popd
PAUSE


code:
Anatomy of a FreeCities event

Type
There are several types of events. They all happen during the end-of-week calculation, AFTER the normal changes to the slaves and all the economic effects (including you messing with your corporation) happened.

Scheduled events
Nonrandom events
Random nonindividual events
Random individual events

The differences between them are almost non-existent. If they happen, they happen in this order, so you could say it's a kind of event priority ordering. The last two events pre-select a slave (in $eventSlave), with the "nonindividual" events using all your slaves while the "individual" events use only non-fuckdolls, typically from your penthouse. When writing your event, you're free to ignore this and choose your own slave or create a new one. By convention, scheduled events tend to go back to the "Scheduled Event" passage so that other such events can run; all the others tend to just skip to the next event category. Nothing in code forces you to do it this way for your events.

Preconditions
Most events have some kind of precondition for when they happen. Scheduled events always happen when their preconditions are true. Nonrandom events happen when their preconditions are true and no other nonrandom events get picked first (so writing a too-often-true precondition is a good way to break the game by blocking any further such events). The two other types of events get put in a pool ($events) if they match the precondition, then at most one event gets pulled from the pool per type.

	NonRandomEvent (26-33)
	<<elseif (_effectiveWeek == 14) && $badC != 1>>
	<<set _valid = $slaves.find(function(s) { return s.curatives > 1 || s.inflationType == "curative"; })>>
	<<if def _valid>>
		<<set $badC = 1, $Event = "bad curatives">>
		<<goto "Generic Plot Events">>
	<<else>>
		<<set $badC = 1>>
		<<goto "Nonrandom Event">>
	<</if>>

If it is week fourteen and the player player hasn't ready seen the event, a check is then made for slaves that either are on curatives or have their implants filled by curatives. If it was sucessful then load the "bad curatives" generic event, if unsucessful set the flag anyway and read from the Nonrandom Event passage.

Immediate effects

Every event can have immediate effects, which happen when the event gets chosen. For most events, those are what should happen if the player ignores the event (by hitting "Continue" or the space bar on the keyboard). Choice effects (see below) can override or roll back those.

		reRecuit (4-19)
		<<if Array.isArray($recruit)>>
	  <<if $cheatMode == 1>>
		<<set $nextButton = "Back", $nextLink = "Nonrandom Event", $returnTo = "Nonrandom Event">> /* if user just clicks spacebar */
		''A random recruit event would have been selected from the following:''

		<<for _i = 0; _i < $recruit.length; _i++>>
			<<print "[[$recruit[_i]|RE recruit][$recruit = $recruit[" + _i + "]]]">>

		<</for>>

[[Go Back to Random Nonindividual Event|Random Nonindividual Event][$eventSlave = 0]]
	  <<else>>
		<<set $recruit = $recruit.random()>>
		<<goto "RE recruit">>
	  <</if>>
	<<else>>

	If cheat mode is enabled and the user presses the space bar go back to the start.

Main event text
The bulk of the writing will be in the main event text. There are quite a few rules to deal with here.
The PC is referred to in the second person singular ("you"), everyone else in third person.
The PC has no direct speech. All the things "you" say are described, not quoted.
A slave can be linked with the macro <<EventNameLink _Slave>>. This allows the player to click on the slave name and view their description, then go back to the event. WARNING: Going back triggers the event passage again, so if your event has immediate effects (see above), make sure to NOT repeat them by setting corresponding flags (this is harder than it sounds in general).

<<SlaveTitle _Slave>> sets $desc (which ends up being a string like "slavegirl", "MILF", "futanari" and so on, depending on slave).

<<SlavePronouns _Slave>> allows you to use the variables $pronoun ("she"/"he"/"it"), $pronounCap ("She"/"He"/"It"), $possessive ("her"/"his"/"its"), $possessiveCap ("Her"/"His"/"Its") and $object ("her"/"him"/"it"). There is NO variable for self-possession ("hers"/"his"/"its") and for "herself", you need to use <<= $object>>self.
One more macro initialises several others and is required when you have a slave speak and want to use direct quotes: <<Enunciate _Slave>> allows you to use <<Master>>, <<WrittenMaster>>, <<says>> (which turns into "lisps" when that's the case) and the lisping replacers <<s>>, <<ss>>, <<S>>, <<c>> and <<z>>.

The text should be about large enough to fit on the screen assuming typical monitor sizes. In terms of visible text, about 1000 words are a fine limit to aim for. There's a lot to keep in mind in terms of different appearances and circumstances, so keep your document with slave variables as a reference nearby.

It's fine - and a part of the normal workflow - to first write an event without any variation, then go through it and vary the text here and there.

Choices
You should keep the amount of choices small, but not too small. About three to five is generally a good number. Choices which can't be taken due to the current situation should be displayed as such ("You lack the funds ...") if they are an obvious choice, hidden when they aren't (for example, in event chains you might want to hide choices if the player didn't do something specific or didn't acquire some specific bit of knowledge). Every choice should be a simple sentence of the form "Do something." followed by a short explanation of the obvious effects.

Choices should also be hidden when they run against the game rules, like for example the setting if the player wants to see "extreme" content.

	diary (313-325)
	<<if $dairyFeedersUpgrade == 1>>
	The milking machines can hold feeders in slaves' mouths and inject drugs into their bodies, ensuring ideal nutrition and production.

    The feeders are
	<<if $dairyFeedersSetting == 2>>
		''industrial.'' [[Moderate|Dairy][$dairyFeedersSetting = 1, $dairyFeedersSettingChanged = -1]]
	<<elseif $dairyFeedersSetting == 1>>
		''active.'' [[Inactive|Dairy][$dairyFeedersSetting = 0]]<<if ($seeExtreme != 0) && ($dairyRestraintsSetting == 2)>> | [[Industrial|Dairy][$dairyFeedersSetting = 2, $dairyFeedersSettingChanged = 1]]<</if>>
	<<else>>
		''inactive.'' [[Active|Dairy][$dairyFeedersSetting = 1]]
	<</if>>
	<<else>>
	$dairyNameCaps is equipped to feed and clean slaves normally. [[Upgrade the milking machines with intubators|Dairy][$cash -= _Tmult1, $dairyFeedersUpgrade = 1]] //Costs ¤_Tmult1 and will increase upkeep costs//
	<</if>>

	In order to enable the industrial feeder option both any of the see extreme content options has be enabled and the restraint's have to be already set to industrial.

Remember that "do nothing" is almost always a choice (it's called "Continue" and can be found on the left side of the screen) so your events don't need this as an extra choice.

Choice text
This should be a short text describing the effects of your choice. Generally shorter than the main text, but all the other things mentioned there apply.
Try to keep surprise buttsex to a minimum.

For example this cut up version of "paternalist encounter" from REFS (l:106-139).

	<<link "Alert your drones and keep walking">>

	<</link>>
	<<if $cash >= 2000>>

<<link "Take the poor slave girl into your custody">>

<<link "Publicly confront the citizen">>

	So here you can either, A) "Alert your drones and keep walking", B) if $cash is above 2000 you can take acquire the slave or C) "Publicly confront the citizen".

Choice effect
A choice doesn't need to have a specific effect. If your event has an immediate effect, remember to take that into account when you decide on the choice's effects.

	<<link "Alert your drones and keep walking">>
	<<replace "#result">>
	You inform $assistantName that you have a slave beater in need of detainment by your security drones, then continue on your way confident in your knowledge that the citizen will soon be in custody.
	<</replace>>
	<</link>>
	<<if $cash >= 2000>>

<<link "Take the poor slave girl into your custody">>
	<<replace "#result">>
	Confronting the citizen is simplicity in itself, he would not dare defy you under threat of arrest by your security drones and is unlikely to garner any sympathy from the public for his degradationist behaviors. As such, you are able to take civil ownership over the poor slave girl and take her into your care with only minimal compensation to the citizen. As you stride away from her former owner with the girl in your arms, she leans over to plant a chaste kiss of thanks on your cheek.
	<<set $cash -= 2000>>
		<<set $activeSlaveOneTimeMaxAge = 22>>
		<<include "Generate New Slave">>
		<<set $activeSlave.origin = "She was taken into your custody from an abusive owner.">>
		<<set $activeSlave.devotion = random(-25,0)>>
		<<set $activeSlave.trust = random(-25,0)>>
		<<set $activeSlave.boobs = random(300,450)>>
		<<set $activeSlave.hips = random(-2,-1)>>
		<<set $activeSlave.butt = random(0,2)>>
		<<set $activeSlave.health = random(10,20)>>
		<<include "New Slave Intro">>
		<<set $nextLink = "AS Dump", $returnTo = "RIE Eligibility Check">>
	<</replace>>
	<</link>> // Taking custody of the girl will cost ¤2000. //
	<</if>>

<<link "Publicly confront the citizen">>
	<<replace "#result">>
	Your walk up to the citizen is not accompanied by shaking ground or tumultuous fanfare, yet the citizen looks as if death itself has come before him. You don't hurt him physically, instead chastising him publicly in front of his fellow peers who begin to cheer their agreement. You end your tirade of verbal abuse with a reminder that although the man is a citizen of your arcology, that does not give him the impunity to shirk the law. To make it clear his next offense will be his last, a brace of your security drones hover behind you threateningly. The crowd that gathered @@.green;approve of your rebuke of the citizen@@.
	<<set $rep += 500>>
	<</replace>>
	<</link>>

	So here the results of the choices are, A) nothing, B) reduce cash by 2000 but acquire the slave C) chastise the citizen for plus 500 rep.