Example project · Builds & projects
Storyteller from source (ESP-IDF)
A real multi-file ESP-IDF C project you can read, edit and rebuild: a line-faithful port of Andrej Karpathy’s llama2.c running the 260K-parameter TinyStories model on the emulated ESP32-S3. The board boots and writes a deterministic story to the serial monitor, then takes your typed prompt and writes another. The Code tab holds the exact C source, CMake files, sdkconfig and partition table the firmware was built from — change the boot banner or the sampling loop, press Compile & run, and the cloud sandbox rebuilds the firmware with ESP-IDF v5.3.5 and boots your build on the bench.
What's on the bench
- Battery
- ESP32-S3
How it's wired
- Battery · pos→ESP32-S3 · vin
- Battery · neg→ESP32-S3 · gnd
The code
The exact ESP-IDF v5.3.5 project the bundled firmware was compiled from — shown in the simulator’s Code tab, where you can edit the sources and press Compile & run to rebuild the board’s firmware in the cloud sandbox.
CMakeLists.txt
cmake_minimum_required(VERSION 3.16) include($ENV{IDF_PATH}/tools/cmake/project.cmake) project(tinyllm-s3)
sdkconfig.defaults
# tinyllm-s3 — deterministic llama2.c inference on the ESP32-S3. CONFIG_IDF_TARGET="esp32s3" # 4 MB flash, custom table: one 3 MB factory app (code + embedded weights). CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" # Full 240 MHz, -O2. No PSRAM on the target board profile. CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y CONFIG_COMPILER_OPTIMIZATION_PERF=y # Data cache stays at the default 32 KB. Do NOT set DATA_CACHE_16KB: with a # 16 KB dcache IDF exposes the unused half of the dcache SRAM as a heap block # aliased at 0x3C000000 (heap/port/esp32s3/memory_layout.c "Level 10"), and # the Breadboard emulator does not model that alias — heap_init then panics # storing into what it maps as flash DROM. CONFIG_ESP32S3_DATA_CACHE_32KB=y # One console only: the default secondary USB-Serial-JTAG console duplicates # every byte in the emulator's merged serial stream. CONFIG_ESP_CONSOLE_SECONDARY_NONE=y # Token generation is a long CPU burst on the main task; the idle task on # core 0 legitimately starves during it. Do not arm the task watchdog. # CONFIG_ESP_TASK_WDT_INIT is not set # The whole model state is static; the main task only needs stack for the # tokenizer scratch and printf. CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 # The 240 KB static KV cache leaves DIRAM tight: move IRAM-optional code to # flash and use the ROM flash driver so the runtime heap keeps a real margin # (~11 KB remained without these; boot needs ~20 KB+). CONFIG_FREERTOS_PLACE_FUNCTIONS_INTO_FLASH=y CONFIG_RINGBUF_PLACE_FUNCTIONS_INTO_FLASH=y CONFIG_HEAP_PLACE_FUNCTION_INTO_FLASH=y CONFIG_SPI_FLASH_ROM_IMPL=y
partitions.csv
# Name, Type, SubType, Offset, Size nvs, data, nvs, 0x9000, 0x6000 phy_init, data, phy, 0xf000, 0x1000 factory, app, factory, 0x10000, 0x300000
main/CMakeLists.txt
idf_component_register( SRCS "tinyllm_main.c" "model_blobs.S" ) # The .S embeds the model via .incbin: hand it absolute paths, and rebuild it # when the blobs change. target_compile_definitions(${COMPONENT_LIB} PRIVATE "CKPT_PATH=\"${CMAKE_CURRENT_SOURCE_DIR}/model/stories260K.bin\"" "TOK_PATH=\"${CMAKE_CURRENT_SOURCE_DIR}/model/tok512.bin\"" ) set_property(SOURCE model_blobs.S APPEND PROPERTY OBJECT_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/model/stories260K.bin" "${CMAKE_CURRENT_SOURCE_DIR}/model/tok512.bin" )
main/tinyllm_main.c
/* * tinyllm-s3 — Llama-2 Transformer inference on the ESP32-S3, ported from * Andrej Karpathy's llama2.c `run.c` (MIT License, Copyright (c) 2023 Andrej * Karpathy; https://github.com/karpathy/llama2.c). The neural-net blocks, * tokenizer, and generation loop below are kept line-faithful to upstream so * the port stays auditable against it. The platform seams differ: * * - checkpoint + tokenizer are EMBED_FILES blobs read in place from the * flash DROM window (the embedded analogue of upstream's read-only mmap; * the 1.03 MB of fp32 weights never touch RAM), * - RunState is statically allocated so the linker proves the SRAM fit at * build time (no PSRAM on this board profile), * - the KV cache is sized to SEQ_CAP=256 (the checkpoint allows 512, but * 512 needs a 640 KB cache — more than the S3's usable internal SRAM), * - sampling is greedy argmax only (temperature 0), so a given prompt * always yields the same story: deterministic by construction, which is * what the emulator golden tests pin, * - timing uses esp_timer, prompts arrive over the UART0 console. * * Model: stories260K (karpathy/tinyllamas, MIT) + tok512 tokenizer. */ #include <ctype.h> #include <math.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include "driver/uart.h" #include "driver/uart_vfs.h" #include "esp_heap_caps.h" #include "esp_timer.h" // ---------------------------------------------------------------------------- // Model constants: stories260K. The checkpoint header is validated against // these at boot; a mismatched blob aborts loudly instead of running garbage. #define MODEL_DIM 64 #define MODEL_HIDDEN 172 #define MODEL_LAYERS 5 #define MODEL_HEADS 8 #define MODEL_KV_HEADS 4 #define MODEL_VOCAB 512 #define MODEL_HEAD_SIZE (MODEL_DIM / MODEL_HEADS) #define MODEL_KV_DIM ((MODEL_DIM * MODEL_KV_HEADS) / MODEL_HEADS) // The checkpoint was trained with seq_len 512, but a 512-position KV cache is // 640 KB of fp32 — beyond the S3's usable SRAM. 192 positions (240 KB) is the // most that links alongside ESP-IDF's own DRAM use while leaving real heap // headroom (256 overflowed dram0_0_seg by 71 KB), and greedy TinyStories // generations hit their EOS well inside it. #define SEQ_CAP 192 #define MAX_PROMPT_CHARS 256 #define MAX_TOKEN_LEN 64 // checked against the tokenizer header at boot // Blobs embedded by model_blobs.S with 16-byte alignment (still asserted at // boot before use). extern const uint8_t ckpt_start[] asm("tinyllm_ckpt_start"); extern const uint8_t ckpt_end[] asm("tinyllm_ckpt_end"); extern const uint8_t tok_start[] asm("tinyllm_tok_start"); extern const uint8_t tok_end[] asm("tinyllm_tok_end"); // ---------------------------------------------------------------------------- // Transformer model (faithful to run.c; pointers are const because the // weights live in flash) typedef struct { int dim; // transformer dimension int hidden_dim; // for ffn layers int n_layers; // number of layers int n_heads; // number of query heads int n_kv_heads; // number of key/value heads (can be < query heads because of multiquery) int vocab_size; // vocabulary size, usually 256 (byte-level) int seq_len; // max sequence length } Config; typedef struct { const float* token_embedding_table; // (vocab_size, dim) const float* rms_att_weight; // (layer, dim) rmsnorm weights const float* rms_ffn_weight; // (layer, dim) const float* wq; // (layer, dim, n_heads * head_size) const float* wk; // (layer, dim, n_kv_heads * head_size) const float* wv; // (layer, dim, n_kv_heads * head_size) const float* wo; // (layer, n_heads * head_size, dim) const float* w1; // (layer, hidden_dim, dim) const float* w2; // (layer, dim, hidden_dim) const float* w3; // (layer, hidden_dim, dim) const float* rms_final_weight; // (dim,) const float* wcls; // (optional) classifier weights for the logits } TransformerWeights; // RunState, statically allocated (upstream callocs these). Sizes are the // exact upstream formulas with seq_len = SEQ_CAP. static float s_x[MODEL_DIM]; static float s_xb[MODEL_DIM]; static float s_xb2[MODEL_DIM]; static float s_hb[MODEL_HIDDEN]; static float s_hb2[MODEL_HIDDEN]; static float s_q[MODEL_DIM]; static float s_att[MODEL_HEADS * SEQ_CAP]; static float s_logits[MODEL_VOCAB]; static float s_key_cache[MODEL_LAYERS * SEQ_CAP * MODEL_KV_DIM]; static float s_value_cache[MODEL_LAYERS * SEQ_CAP * MODEL_KV_DIM]; static Config g_config; // header values from the checkpoint (seq_len = 512) static TransformerWeights g_weights; static void memory_map_weights(TransformerWeights* w, Config* p, const float* ptr, int shared_weights) { int head_size = p->dim / p->n_heads; unsigned long long n_layers = p->n_layers; w->token_embedding_table = ptr; ptr += p->vocab_size * p->dim; w->rms_att_weight = ptr; ptr += n_layers * p->dim; w->wq = ptr; ptr += n_layers * p->dim * (p->n_heads * head_size); w->wk = ptr; ptr += n_layers * p->dim * (p->n_kv_heads * head_size); w->wv = ptr; ptr += n_layers * p->dim * (p->n_kv_heads * head_size); w->wo = ptr; ptr += n_layers * (p->n_heads * head_size) * p->dim; w->rms_ffn_weight = ptr; ptr += n_layers * p->dim; w->w1 = ptr; ptr += n_layers * p->dim * p->hidden_dim; w->w2 = ptr; ptr += n_layers * p->hidden_dim * p->dim; w->w3 = ptr; ptr += n_layers * p->dim * p->hidden_dim; w->rms_final_weight = ptr; ptr += p->dim; ptr += p->seq_len * head_size / 2; // skip what used to be freq_cis_real (for RoPE) ptr += p->seq_len * head_size / 2; // skip what used to be freq_cis_imag (for RoPE) w->wcls = shared_weights ? w->token_embedding_table : ptr; } static void build_transformer(void) { if (((uintptr_t)ckpt_start & 3) != 0) { printf("[tinyllm] FATAL: checkpoint blob not 4-byte aligned\n"); abort(); } memcpy(&g_config, ckpt_start, sizeof(Config)); int shared_weights = g_config.vocab_size > 0 ? 1 : 0; g_config.vocab_size = abs(g_config.vocab_size); if (g_config.dim != MODEL_DIM || g_config.hidden_dim != MODEL_HIDDEN || g_config.n_layers != MODEL_LAYERS || g_config.n_heads != MODEL_HEADS || g_config.n_kv_heads != MODEL_KV_HEADS || g_config.vocab_size != MODEL_VOCAB) { printf("[tinyllm] FATAL: checkpoint header does not match compiled model\n"); abort(); } // Sanity: the blob must hold exactly header + all weight floats. unsigned long long head_size = MODEL_HEAD_SIZE; unsigned long long want = (unsigned long long)g_config.vocab_size * MODEL_DIM + // token embedding 2ull * MODEL_LAYERS * MODEL_DIM + // rms att+ffn 2ull * MODEL_LAYERS * MODEL_DIM * MODEL_DIM + // wq, wo 2ull * MODEL_LAYERS * MODEL_DIM * MODEL_KV_DIM + // wk, wv 3ull * MODEL_LAYERS * MODEL_DIM * MODEL_HIDDEN + // w1, w2, w3 MODEL_DIM + // rms final (unsigned long long)g_config.seq_len * head_size + // legacy freq_cis (shared_weights ? 0ull : (unsigned long long)MODEL_VOCAB * MODEL_DIM); unsigned long long have = (unsigned long long)(ckpt_end - ckpt_start); if (have != sizeof(Config) + want * sizeof(float)) { printf("[tinyllm] FATAL: checkpoint size %llu != expected %llu\n", have, sizeof(Config) + want * sizeof(float)); abort(); } const float* weights_ptr = (const float*)(ckpt_start + sizeof(Config)); memory_map_weights(&g_weights, &g_config, weights_ptr, shared_weights); } // ---------------------------------------------------------------------------- // neural net blocks; the dynamics of the Transformer (verbatim from run.c, // modulo const) static void rmsnorm(float* o, const float* x, const float* weight, int size) { float ss = 0.0f; for (int j = 0; j < size; j++) { ss += x[j] * x[j]; } ss /= size; ss += 1e-5f; ss = 1.0f / sqrtf(ss); for (int j = 0; j < size; j++) { o[j] = weight[j] * (ss * x[j]); } } static void softmax(float* x, int size) { float max_val = x[0]; for (int i = 1; i < size; i++) { if (x[i] > max_val) { max_val = x[i]; } } float sum = 0.0f; for (int i = 0; i < size; i++) { x[i] = expf(x[i] - max_val); sum += x[i]; } for (int i = 0; i < size; i++) { x[i] /= sum; } } static void matmul(float* xout, const float* x, const float* w, int n, int d) { // W (d,n) @ x (n,) -> xout (d,) // by far the most amount of time is spent inside this little function for (int i = 0; i < d; i++) { float val = 0.0f; for (int j = 0; j < n; j++) { val += w[i * n + j] * x[j]; } xout[i] = val; } } static float* forward(int token, int pos) { Config* p = &g_config; TransformerWeights* w = &g_weights; float* x = s_x; int dim = p->dim; int kv_dim = (p->dim * p->n_kv_heads) / p->n_heads; int kv_mul = p->n_heads / p->n_kv_heads; int hidden_dim = p->hidden_dim; int head_size = dim / p->n_heads; const float* content_row = w->token_embedding_table + token * dim; memcpy(x, content_row, dim * sizeof(*x)); for (unsigned long long l = 0; l < p->n_layers; l++) { rmsnorm(s_xb, x, w->rms_att_weight + l * dim, dim); // key and value point to the kv cache (seq stride is SEQ_CAP here) int loff = l * SEQ_CAP * kv_dim; float* k = s_key_cache + loff + pos * kv_dim; float* v = s_value_cache + loff + pos * kv_dim; matmul(s_q, s_xb, w->wq + l * dim * dim, dim, dim); matmul(k, s_xb, w->wk + l * dim * kv_dim, dim, kv_dim); matmul(v, s_xb, w->wv + l * dim * kv_dim, dim, kv_dim); // RoPE relative positional encoding: complex-valued rotate q and k in each head for (int i = 0; i < dim; i += 2) { int head_dim = i % head_size; float freq = 1.0f / powf(10000.0f, head_dim / (float)head_size); float val = pos * freq; float fcr = cosf(val); float fci = sinf(val); int rotn = i < kv_dim ? 2 : 1; // how many vectors? 2 = q & k, 1 = q only for (int vi = 0; vi < rotn; vi++) { float* vec = vi == 0 ? s_q : k; float v0 = vec[i]; float v1 = vec[i + 1]; vec[i] = v0 * fcr - v1 * fci; vec[i + 1] = v0 * fci + v1 * fcr; } } // multihead attention. iterate over all heads for (int h = 0; h < p->n_heads; h++) { float* q = s_q + h * head_size; float* att = s_att + h * SEQ_CAP; for (int t = 0; t <= pos; t++) { const float* kt = s_key_cache + loff + t * kv_dim + (h / kv_mul) * head_size; float score = 0.0f; for (int i = 0; i < head_size; i++) { score += q[i] * kt[i]; } score /= sqrtf(head_size); att[t] = score; } softmax(att, pos + 1); float* xb = s_xb + h * head_size; memset(xb, 0, head_size * sizeof(float)); for (int t = 0; t <= pos; t++) { const float* vt = s_value_cache + loff + t * kv_dim + (h / kv_mul) * head_size; float a = att[t]; for (int i = 0; i < head_size; i++) { xb[i] += a * vt[i]; } } } matmul(s_xb2, s_xb, w->wo + l * dim * dim, dim, dim); for (int i = 0; i < dim; i++) { x[i] += s_xb2[i]; } rmsnorm(s_xb, x, w->rms_ffn_weight + l * dim, dim); // Now for FFN in PyTorch we have: self.w2(F.silu(self.w1(x)) * self.w3(x)) matmul(s_hb, s_xb, w->w1 + l * dim * hidden_dim, dim, hidden_dim); matmul(s_hb2, s_xb, w->w3 + l * dim * hidden_dim, dim, hidden_dim); // SwiGLU non-linearity for (int i = 0; i < hidden_dim; i++) { float val = s_hb[i]; val *= (1.0f / (1.0f + expf(-val))); val *= s_hb2[i]; s_hb[i] = val; } matmul(s_xb, s_hb, w->w2 + l * dim * hidden_dim, hidden_dim, dim); for (int i = 0; i < dim; i++) { x[i] += s_xb[i]; } } rmsnorm(x, x, w->rms_final_weight, dim); matmul(s_logits, x, w->wcls, p->dim, p->vocab_size); return s_logits; } // ---------------------------------------------------------------------------- // The Byte Pair Encoding (BPE) Tokenizer that translates strings <-> tokens // (faithful to run.c; file IO replaced by an in-place parse of the embedded // tok512.bin into a static arena) typedef struct { const char* str; int id; } TokenIndex; static const char* g_vocab[MODEL_VOCAB]; static float g_vocab_scores[MODEL_VOCAB]; static TokenIndex g_sorted_vocab[MODEL_VOCAB]; static unsigned int g_max_token_length; static unsigned char g_byte_pieces[512]; // stores all single-byte strings static char g_vocab_arena[8192]; static int compare_tokens(const void* a, const void* b) { return strcmp(((const TokenIndex*)a)->str, ((const TokenIndex*)b)->str); } static void build_tokenizer(void) { for (int i = 0; i < 256; i++) { g_byte_pieces[i * 2] = (unsigned char)i; g_byte_pieces[i * 2 + 1] = '\0'; } const uint8_t* p = tok_start; const uint8_t* end = tok_end; memcpy(&g_max_token_length, p, sizeof(int)); p += sizeof(int); if (g_max_token_length > MAX_TOKEN_LEN) { printf("[tinyllm] FATAL: tokenizer max_token_length %u > %d\n", g_max_token_length, MAX_TOKEN_LEN); abort(); } char* arena = g_vocab_arena; char* arena_end = g_vocab_arena + sizeof(g_vocab_arena); for (int i = 0; i < MODEL_VOCAB; i++) { int len; if (p + sizeof(float) + sizeof(int) > end) { printf("[tinyllm] FATAL: tokenizer blob truncated at token %d\n", i); abort(); } memcpy(&g_vocab_scores[i], p, sizeof(float)); p += sizeof(float); memcpy(&len, p, sizeof(int)); p += sizeof(int); if (len < 0 || p + len > end || arena + len + 1 > arena_end) { printf("[tinyllm] FATAL: tokenizer blob invalid at token %d\n", i); abort(); } memcpy(arena, p, len); arena[len] = '\0'; g_vocab[i] = arena; arena += len + 1; p += len; } if (p != end) { printf("[tinyllm] FATAL: tokenizer blob has %d trailing bytes\n", (int)(end - p)); abort(); } for (int i = 0; i < MODEL_VOCAB; i++) { g_sorted_vocab[i].str = g_vocab[i]; g_sorted_vocab[i].id = i; } qsort(g_sorted_vocab, MODEL_VOCAB, sizeof(TokenIndex), compare_tokens); } static const char* decode(int prev_token, int token) { const char* piece = g_vocab[token]; // following BOS (1) token, sentencepiece decoder strips any leading whitespace if (prev_token == 1 && piece[0] == ' ') { piece++; } // careful, some tokens designate raw bytes, and look like e.g. '<0x01>' unsigned char byte_val; if (sscanf(piece, "<0x%02hhX>", &byte_val) == 1) { piece = (const char*)g_byte_pieces + byte_val * 2; } return piece; } static void safe_printf(const char* piece) { // piece might be a raw byte token, and we only want to print printable // chars or whitespace if (piece == NULL) { return; } if (piece[0] == '\0') { return; } if (piece[1] == '\0') { unsigned char byte_val = piece[0]; if (!(isprint(byte_val) || isspace(byte_val))) { return; // bad byte, don't print it } } printf("%s", piece); } static int str_lookup(const char* str, const TokenIndex* sorted_vocab, int vocab_size) { TokenIndex tok = {.str = str}; const TokenIndex* res = bsearch(&tok, sorted_vocab, vocab_size, sizeof(TokenIndex), compare_tokens); return res != NULL ? res->id : -1; } static void encode(const char* text, int8_t bos, int8_t eos, int* tokens, int* n_tokens) { // encode the string text (input) into an upper-bound preallocated tokens[] // array; bos != 0 prepends BOS (=1), eos != 0 appends EOS (=2) static char str_buffer[MAX_TOKEN_LEN * 2 + 1 + 2]; size_t str_len = 0; *n_tokens = 0; if (bos) tokens[(*n_tokens)++] = 1; // add_dummy_prefix is true by default: prepend a dummy prefix token to the // input string, but only if text != "" if (text[0] != '\0') { int dummy_prefix = str_lookup(" ", g_sorted_vocab, MODEL_VOCAB); tokens[(*n_tokens)++] = dummy_prefix; } // process the raw (UTF-8) byte sequence of the input string for (const char* c = text; *c != '\0'; c++) { if ((*c & 0xC0) != 0x80) { str_len = 0; } str_buffer[str_len++] = *c; str_buffer[str_len] = '\0'; if ((*(c + 1) & 0xC0) == 0x80 && str_len < 4) { continue; } int id = str_lookup(str_buffer, g_sorted_vocab, MODEL_VOCAB); if (id != -1) { tokens[(*n_tokens)++] = id; } else { // byte_fallback encoding: encode each byte as a token (+3 because // the first 3 vocab elements are <unk>, <s>, </s>) for (size_t i = 0; i < str_len; i++) { tokens[(*n_tokens)++] = (unsigned char)str_buffer[i] + 3; } } str_len = 0; } // merge the best consecutive pair each iteration, per the vocab_scores while (1) { float best_score = -1e10; int best_id = -1; int best_idx = -1; for (int i = 0; i < (*n_tokens - 1); i++) { snprintf(str_buffer, sizeof(str_buffer), "%s%s", g_vocab[tokens[i]], g_vocab[tokens[i + 1]]); int id = str_lookup(str_buffer, g_sorted_vocab, MODEL_VOCAB); if (id != -1 && g_vocab_scores[id] > best_score) { best_score = g_vocab_scores[id]; best_id = id; best_idx = i; } } if (best_idx == -1) { break; } tokens[best_idx] = best_id; for (int i = best_idx + 1; i < (*n_tokens - 1); i++) { tokens[i] = tokens[i + 1]; } (*n_tokens)--; } if (eos) tokens[(*n_tokens)++] = 2; } // ---------------------------------------------------------------------------- // sampling: greedy argmax only (temperature 0) — deterministic by construction static int sample_argmax(const float* probabilities, int n) { int max_i = 0; float max_p = probabilities[0]; for (int i = 1; i < n; i++) { if (probabilities[i] > max_p) { max_i = i; max_p = probabilities[i]; } } return max_i; } // ---------------------------------------------------------------------------- // generation loop (faithful to run.c generate(); esp_timer for tok/s) static void generate(const char* prompt, int steps) { static int prompt_tokens[MAX_PROMPT_CHARS + 3]; // +3 for '\0', ?BOS, ?EOS int num_prompt_tokens = 0; encode(prompt, 1, 0, prompt_tokens, &num_prompt_tokens); if (num_prompt_tokens < 1) { printf("[tinyllm] FATAL: expected at least 1 prompt token\n"); abort(); } int64_t start = 0; // timer starts after the first iteration, as upstream int next; int token = prompt_tokens[0]; int pos = 0; while (pos < steps) { float* logits = forward(token, pos); if (pos < num_prompt_tokens - 1) { next = prompt_tokens[pos + 1]; } else { next = sample_argmax(logits, g_config.vocab_size); } pos++; // data-dependent terminating condition: the BOS (=1) token delimits sequences if (next == 1) { break; } const char* piece = decode(token, next); safe_printf(piece); fflush(stdout); token = next; if (start == 0) { start = esp_timer_get_time(); } } printf("\n"); if (pos > 1) { int64_t end = esp_timer_get_time(); double secs = (end - start) / 1e6; printf("[tinyllm] %d tokens, achieved tok/s: %.2f\n", pos, (pos - 1) / secs); } } // ---------------------------------------------------------------------------- // console: blocking line input over UART0 static void console_init(void) { uart_driver_install(UART_NUM_0, 256, 0, 0, NULL, 0); uart_vfs_dev_use_driver(UART_NUM_0); uart_vfs_dev_port_set_rx_line_endings(UART_NUM_0, ESP_LINE_ENDINGS_CR); } static void read_prompt(char* buf, size_t cap) { if (fgets(buf, cap, stdin) == NULL) { buf[0] = '\0'; return; } buf[strcspn(buf, "\r\n")] = '\0'; } void app_main(void) { setvbuf(stdout, NULL, _IONBF, 0); printf("\n[tinyllm] llama2.c stories260K on ESP32-S3 (fp32, flash-resident weights)\n"); build_transformer(); build_tokenizer(); printf("[tinyllm] model: dim=%d hidden=%d layers=%d heads=%d kv_heads=%d vocab=%d seq=%d (cap %d)\n", g_config.dim, g_config.hidden_dim, g_config.n_layers, g_config.n_heads, g_config.n_kv_heads, g_config.vocab_size, g_config.seq_len, SEQ_CAP); printf("[tinyllm] free heap: %u bytes\n", (unsigned)esp_get_free_heap_size()); const char* boot_prompt = "Once upon a time"; printf("[tinyllm] boot story (greedy, prompt \"%s\"):\n\n", boot_prompt); generate(boot_prompt, SEQ_CAP); printf("[tinyllm] boot story done\n"); console_init(); static char line[MAX_PROMPT_CHARS]; while (1) { printf("\nprompt> "); read_prompt(line, sizeof(line)); printf("[prompt: %s]\n\n", line); generate(line[0] != '\0' ? line : "Once upon a time", SEQ_CAP); } }
main/model_blobs.S
/* * Embedded model assets with guaranteed alignment. IDF's EMBED_FILES gives * no alignment guarantee (the stories260K blob landed on an odd address), * and the fp32 checkpoint is read in place from DROM with 4-byte loads. * CKPT_PATH / TOK_PATH are absolute paths injected by main/CMakeLists.txt. */ .section .rodata.tinyllm_model, "a" .balign 16 .global tinyllm_ckpt_start tinyllm_ckpt_start: .incbin CKPT_PATH .global tinyllm_ckpt_end tinyllm_ckpt_end: .balign 16 .global tinyllm_tok_start tinyllm_tok_start: .incbin TOK_PATH .global tinyllm_tok_end tinyllm_tok_end:
