Iterating Strings — Professional Level (Internals, Compiler, Memory, Assembly)¶

// runtime/string.go
type stringStruct struct {
    str unsafe.Pointer // pointer to bytes
    len int            // byte length
}

// Identical to reflect.StringHeader:
// type StringHeader struct { Data uintptr; Len int }

// A string literal in Go is stored in the read-only data segment (.rodata)
// Converting string → []byte always copies (string is immutable)
// Substrings share the same backing memory — no copy

// unicode/utf8/utf8.go — the 'first' lookup table
// Each byte maps to encoding info:
// - top 3 bits: rune type (ASCII, 2-byte, 3-byte, 4-byte, invalid)
// - bottom 3 bits: sequence length
var first = [256]uint8{
    //   0   1   2   3   4   5   6   7   8   9   a   b   c   d   e   f
    as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x00-0x0f
    // ... 128 ASCII entries (0x00-0x7F) all map to 'as' (ASCII single byte)
    // ... continuation bytes (0x80-0xBF) map to 'xx' (invalid lead)
    // ... 2-byte leads (0xC0-0xDF) map to 'x1' (2-byte sequence)
    // ... 3-byte leads (0xE0-0xEF) map to various 3-byte types
    // ... 4-byte leads (0xF0-0xF7) map to various 4-byte types
    // ... invalid (0xF8-0xFF) map to 'xx'
}

func countRunes(s string) int { count := 0; for range s { count++ }; return count }

// strings/strings_amd64.go (internal assembly)
// strings.Index uses Boyer-Moore-Horspool with SSE2/AVX2
// strings.Count uses SSE2 for counting bytes

// For ASCII-heavy workloads, using strings.Index is faster than manual range
// because it operates at the hardware level

// Example: finding a byte in a large string
import "strings"

func findByte(s string, b byte) int {
    return strings.IndexByte(s, b)
    // Internally: loads 16 bytes at a time via SSE2 PCMPEQB instruction
    // ~10x faster than manual range for large strings
}

package main

import (
    "fmt"
    "unsafe"
)

// Zero-copy []byte from string (READ ONLY — DO NOT MODIFY!)
func unsafeBytes(s string) []byte {
    if s == "" { return nil }
    return unsafe.Slice(unsafe.StringData(s), len(s))
    // unsafe.StringData(s) returns *byte pointer to first byte
    // This is valid in Go 1.20+
}

// Zero-copy string from []byte (string must not modify underlying bytes)
func unsafeString(b []byte) string {
    if len(b) == 0 { return "" }
    return unsafe.String(&b[0], len(b))
    // Go 1.20+: unsafe.String(*byte, int) string
}

func main() {
    s := "Hello, World!"
    b := unsafeBytes(s)
    fmt.Printf("b[0]=%d\n", b[0]) // 72 — same bytes as s
    // DO NOT: b[0] = 99 — this would corrupt the string literal!
}

// The Go compiler interns string literals:
a := "hello"
b := "hello"
// a and b may point to the same memory — implementation defined

// You can verify:
import "unsafe"
sa := (*[2]uintptr)(unsafe.Pointer(&a))
sb := (*[2]uintptr)(unsafe.Pointer(&b))
fmt.Println(sa[0] == sb[0]) // often true for literals

// runtime.stringinterner is used internally for some cases
// but user code should not rely on interning behavior

// When strings are used as map keys, Go hashes them:
// runtime/hash.go

// For strings, Go uses AES-based hash on platforms with AES hardware:
// - x86: uses AES-NI instructions
// - ARM: uses software AES or fallback

// The hash of a string depends on:
// 1. The string contents (all bytes)
// 2. A per-map random seed (hash0 in hmap)

// This means:
// - Two programs hashing "hello" may get different values (seed differs)
// - Security: prevents hash-flooding via random seed

// The compiler optimizes short string comparisons to integer comparisons:
func equal4(a, b string) bool {
    if len(a) != 4 || len(b) != 4 { return false }
    return a == b
    // Compiler may emit: load 4 bytes as uint32, compare
    // No byte-by-byte loop needed
}

// For variable-length comparisons, runtime uses:
// - memcmp for same-length strings (optimized per platform)
// - Early exit on length mismatch

// utf8.EncodeRune writes one rune into a byte slice
// Understanding the encoding for professional use:

func encodeRune(r rune) []byte {
    switch {
    case r < 0x80:
        return []byte{byte(r)}
    case r < 0x800:
        return []byte{
            0xC0 | byte(r>>6),
            0x80 | byte(r&0x3F),
        }
    case r < 0x10000:
        return []byte{
            0xE0 | byte(r>>12),
            0x80 | byte((r>>6)&0x3F),
            0x80 | byte(r&0x3F),
        }
    default:
        return []byte{
            0xF0 | byte(r>>18),
            0x80 | byte((r>>12)&0x3F),
            0x80 | byte((r>>6)&0x3F),
            0x80 | byte(r&0x3F),
        }
    }
}

// Strings are NOT garbage collected separately
// They are either:
// 1. String literals: in .rodata segment, never GC'd
// 2. Dynamically created: the backing byte array is heap-allocated
//    and GC'd when no string references it

// Problem: Large string keeps large allocation alive
largeData := readHuge() // 100MB string
small := largeData[:10] // substring — keeps 100MB alive!

// Fix: explicit copy
small := string([]byte(largeData[:10])) // new 10-byte allocation, releases large

// strings.Clone (Go 1.20) for explicit copy:
import "strings"
small = strings.Clone(largeData[:10])

package main

import (
    "fmt"
    "reflect"
    "unsafe"
)

// Inspecting string internals via reflect
func inspectString(s string) {
    h := (*reflect.StringHeader)(unsafe.Pointer(&s))
    fmt.Printf("Data: %x\n", h.Data)
    fmt.Printf("Len:  %d\n", h.Len)

    // Read bytes directly from Data pointer
    bytes := unsafe.Slice((*byte)(unsafe.Pointer(h.Data)), h.Len)
    fmt.Printf("Bytes: % X\n", bytes)
}

func main() {
    inspectString("Hello, 世界")
}

//go:nosplit
func criticalStringProcess(s string) int {
    // This function cannot grow the stack
    // Range over large strings in nosplit functions is dangerous:
    // - Each multi-byte rune call hits utf8.DecodeRuneInString
    // - That function may have its own stack requirements
    // - Keep nosplit functions to simple ASCII byte loops
    count := 0
    for i := 0; i < len(s); i++ { // byte loop — safe in nosplit
        if s[i] > 32 { count++ }
    }
    return count
}

package main

import (
    "strings"
    "testing"
    "unicode/utf8"
)

var testStr = strings.Repeat("Hello, 世界! 😀", 1000)

func BenchmarkRange(b *testing.B) {
    for n := 0; n < b.N; n++ {
        count := 0
        for range testStr { count++ }
    }
}

func BenchmarkRuneSlice(b *testing.B) {
    for n := 0; n < b.N; n++ {
        runes := []rune(testStr)
        count := len(runes)
        _ = count
    }
}

func BenchmarkManualUTF8(b *testing.B) {
    for n := 0; n < b.N; n++ {
        count := 0
        for i := 0; i < len(testStr); {
            _, size := utf8.DecodeRuneInString(testStr[i:])
            count++
            i += size
        }
    }
}

func BenchmarkRuneCount(b *testing.B) {
    for n := 0; n < b.N; n++ {
        _ = utf8.RuneCountInString(testStr)
    }
}

// Results (approximate, 10K char string with mix of ASCII and multi-byte):
// BenchmarkRange:      ~5 μs/op, 0 alloc
// BenchmarkRuneSlice:  ~15 μs/op, 1 alloc (40KB)
// BenchmarkManualUTF8: ~8 μs/op, 0 alloc
// BenchmarkRuneCount:  ~3 μs/op, 0 alloc (uses SIMD internally)