Code explanation

Can somebody explain why this code outputs what it does?

// tested with Core 2 Duo, Core 2 Quad and Xeon
// tested with gcc4.1.2 gcc4.4.3 and gcc4.6.1
// compile with: gcc -O0 -m32
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <setjmp.h>

jmp_buf p;
void (*q)();

const char *data =
    "\x8b\x44\x24\x04\x8b\x5c\x24\x08"
    "\x8b\x00\x8b\x1b\x31\xc3\x31\xd8"
    "\x31\xc3\x8b\x4c\x24\x04\x89\x01"
    "\x8b\x4c\x24\x08\x89\x19\xc3\x90"
    "\x55\x89\xe5\x8b\x45\x04\xc9\xc3"
    "\x55\x90\x90\x89\xe5\x90\x90\x90"
    "\x8b\x45\x08\x89\x45\x04\xc9\xc3"
    "\x60\x09\x0e\x13\x14\x01\x0c\x0c"
    "\xc0\x07\x05\x0e\x14\x0f\x0f\x60"
    "\x00\x67\x6f\x74\x6f\x20\x63\x6f"
    "\x6e\x73\x69\x64\x65\x72\x65\x64"
    "\x20\x68\x61\x72\x6d\x66\x75\x6c"
    "\x6c\x00\x90\x90\x1c\x1b\x0a\x20";

int f(int x)
{
    static int b = 0; static int s = 0;
    int a = 0, t;
    if (!s) {
        a = b; b = x;
    } else {
        a = x; t = b;
        do {
            a ^= b;
            b = (a^b) & b;
            b <<= 1;
        } while (b);
        b = t;
    }
    s = (s+1) % 2;
    return a;
}

int g(int i, int *j)
{
    *j = i;
    i = (int) putchar;
    if (*j == (48 << 1)) 
        __asm volatile (
                "movl 8(%ebp),%eax;"
                "leave;"
                "ret"
                );
    return (int) puts;
}

void h(int i)
{
    int b;
    q = (void(*)()) g(i++[data],&b);
    for (f(b);*(data+i)!=b;++i,f(b))
        q(f(i[data])%0xff);
}

void sh(int s)
{
    if (s == 010)
        ((void(*)())g(s,&s))("F");
    longjmp(p,s);
}

int main(void)
{
    int base, addr = 0xffffffff, offs = 16;
    int a = 11, b = 32, i = 25;
    int s = 8, t = 1, u = 4;
    ((void(*)()) data)(&a,&b);
    ((void(*)()) data)(&b,&t);
    ((void(*)()) data)(&t,&s);
    addr ^= a;
       a ^= addr;
    addr ^= a;
    base = ((int(*)())data+addr)();
    if (a == -1)
        goto over;
    puts("A");

    base = (1<<3) | ((f(addr) + f(offs)) & ~0xff);
    h(base+addr+offs);
    exit(0);

over:
    signal(t,sh);signal(s,sh);signal(u,sh);

    if (!(s = setjmp(p))) {
        q = (void(*)()) g(0x30, &a);
        q(data + a + i);
        s = a / (b-1);
        puts("B");
    } else if (s == 0xb) {
        puts("C");
        ((int(*)(int)) data+addr+(offs/2))(base);
    } else {
        puts("D");
        *((int*) base+s) = 0xffffffff;
    }
   
    puts("E");
    return 1;
}

>>1-6 '
>inb4 it relies on unsafe code
>inb4 it relies on undefined behaviour
>inb4 autism


const char *data =
        "\x8b\x44\x24\x04\x8b\x5c\x24\x08"
        "\x8b\x00\x8b\x1b\x31\xc3\x31\xd8"
        "\x31\xc3\x8b\x4c\x24\x04\x89\x01"
        "\x8b\x4c\x24\x08\x89\x19\xc3\x90"
        "\x55\x89\xe5\x8b\x45\x04\xc9\xc3"
        "\x55\x90\x90\x89\xe5\x90\x90\x90"
        "\x8b\x45\x08\x89\x45\x04\xc9\xc3"
        "\x60\x09\x0e\x13\x14\x01\x0c\x0c"
        "\xc0\x07\x05\x0e\x14\x0f\x0f\x60"
        "\x00\x67\x6f\x74\x6f\x20\x63\x6f"
        "\x6e\x73\x69\x64\x65\x72\x65\x64"
        "\x20\x68\x61\x72\x6d\x66\x75\x6c"
        "\x6c\x00\x90\x90\x1c\x1b\x0a\x20";
Contains three functions and some character data and "encoded" character data.


int f(int x)
{
        static int b = 0; static int s = 0;
        int a = 0, t;
        if (!s) {
                a = b; b = x;
        } else {
                a = x; t = b;
                do {
                        a ^= b;
                        b = (a^b) & b;
                        b <<= 1;
                } while (b);
                b = t;
        }
        s = (s+1) % 2;
        return a;
}
Adds two integers, it temporarily stores the first value and then adds the second and returns the sum on each even numbered call.


int g(int i, int *j)
{
        *j = i;
        i = (int) putchar;
        if (*j == (48 << 1)) 
                __asm volatile (
                                "movl 8(%ebp),%eax;"
                                "leave;"
                                "ret"
                                );
        return (int) puts;
}
g dereferences j and set it to i, returns the address of putchar if i == 96, else it returns the address of puts.


((void(*)()) data)(&a,&b);
((void(*)()) data)(&b,&t);
((void(*)()) data)(&t,&s);
Swaps a and b, b and t, s and t using XOR swap. a = 32, b = 1, t = 8, s = 11


base = ((int(*)())data+addr)();
Takes the return address and returns it, i.e. the address of the next instruction.


addr ^= a;
a ^= addr;
addr ^= a;
Swaps a and addr, a is now -1.


if (a == -1)
    goto over;
is now true.

signal(t,sh);signal(s,sh);signal(u,sh); sets up signal handlers for SIGFPE SIGSEGV and SIGILL.


if (!(s = setjmp(p))) { // true the first time
    q = (void(*)()) g(0x30, &a); // sets a to 48, puts is returned
    q(data + a + i); // calls puts with address of data+a+i = data+48+25
    s = a / (b-1); // divides by zero
    puts("B");
} [...]
Prints goto considered harmfull, raises a SIGFPE.


void sh(int s)
{
        if (s == 010)
                ((void(*)())g(s,&s))("F");
        longjmp(p,s);
}
The signal handler, calls puts with "F". Then longjumps back to the setjump point.


} else {
    puts("D");
    *((int*) base+s) = 0xffffffff;
}
Prints "D" and raises a SIGSEGV by attempting to write to base+s (which in this case is in the middle of main, read-only memory.

Back to the signal handler and then to

} else if (s == 0xb) {
    puts("C");
    ((int(*)(int)) data+addr+(offs/2))(base);
}
Which prints "C" and then calls one of the functions stored in data with base as argument. This function works as longjmp, by replacing the return address with the value of base, which is back to:


        if (a == -1)
                goto over;
        puts("A");
 
        base = (1<<3) | ((f(addr) + f(offs)) & ~0xff);
        h(base+addr+offs);
        exit(0);
The value of a is now different from -1, so it prints "A" then calls h after some magic fucking with base and addr and offs. Ultimately exiting.


void h(int i)
{
        int b;
        q = (void(*)()) g(i++[data],&b);
        for (f(b);*(data+i)!=b;++i,f(b))
                q(f(i[data])%0xff);
}
Basically, it prints the "encoded" string, which says "install gentoo"