int clone(void (*fn)(void *), void *arg, void *stack,
          int n_pages, int flags);

int
kclone(uint64 fn, uint64 arg, uint64 stack, int n_pages, int flags)
{
  int i, pid;
  struct proc *np;
  struct proc *p = myproc();
 
  if ((np = allocproc()) == 0)
    return -1;
 
  if (flags & CLONE_VM) { // === thread path ===
    // mm_struct는 공유
    struct mm_struct *m = p->mm;
    if(mm_get(m) < 0)
      goto fail;
    np->mm = m;
 
    // trapframe은 구분
    uint64 tf_va = find_free_tf_va(m->pagetable);
    if(tf_va == 0)
      goto fail;
    np->tf_va = tf_va;
    if (mappages(m->pagetable, tf_va, PGSIZE, (uint64)np->trapframe, PTE_R | PTE_W ) == -1) {
      goto fail;
    };
    *(np->trapframe) = *(p->trapframe);
 
    // thread group 설정
    np->group_leader = p->group_leader;
    np->tgid = p->tgid;
    np->ustack = (void*)stack;
 
    np->trapframe->epc = fn;
    np->trapframe->a0 = arg;
    np->trapframe->sp = stack + n_pages * PGSIZE; // 초기 stack pointer이므로
  }
  
// ------------------------------------------------
 

struct mm_struct *m = p->mm;
mm_get(m);
np->mm = m;

int
mm_get(struct mm_struct *mm)
{
  acquire(&mm->lock);
 
  if(mm->refcount < 1)
    panic("mm_get: bad refcount");
 
  if(mm->refcount >= NTHREAD){
    release(&mm->lock);
    return -1;
  }
 
  mm->refcount++;
  release(&mm->lock);
  return 0;
}

uint64 tf_va = find_free_tf_va(m->pagetable);
mappages(m->pagetable, tf_va, PGSIZE, (uint64)np->trapframe, ...);
*(np->trapframe) = *(p->trapframe);

uint64
find_free_tf_va(pagetable_t pagetable)
{
  for (int i = 0; i < NTHREAD; i++) {
    uint64 loc = TRAPFRAME - i * PGSIZE; 
    if (ismapped(pagetable, loc)) continue;
    return loc;
  }
  return 0;
}

np->trapframe->epc = fn;
np->trapframe->a0 = arg;
np->trapframe->sp = stack + n_pages * PGSIZE;

np->group_leader = p->group_leader;
np->tgid = p->tgid;
np->ustack = (void*)stack;

static void
freeproc(struct proc *p)
{
  struct mm_struct *m = p->mm;
 
  if(p->trapframe) {
    int freed_by_unmap = 0;
 
    if(m && m->pagetable && p->tf_va) {
      acquire(&m->lock);
      if(ismapped(m->pagetable, p->tf_va)) {
        uvmunmap(m->pagetable, p->tf_va, 1, 1); // VA(trapframe) + PA 정리
        freed_by_unmap = 1;
      }
      release(&m->lock);
    }
 
    if(!freed_by_unmap)
      kfree(p->trapframe);
 
    p->trapframe = 0;
  }
 
  p->tf_va = 0;
 
  if(m) {
    mm_put(m);
    p->mm = 0;
  }
 
  // PCB정리
  cleantask(p);
}

void
mm_put(struct mm_struct *mm)
{
  acquire(&mm->lock);
  mm->refcount--;
  int ref = mm->refcount;
  release(&mm->lock);
 
  if (ref == 0) {
    if(mm->pagetable)
      proc_freepagetable(mm->pagetable, mm->sz);
    kfree(mm);
  }
}

static void
cleantask(struct proc *p) 
{
  p->pid = 0;
  p->parent = 0;
  p->name[0] = 0;
  p->chan = 0;
  p->killed = 0;
  p->xstate = 0;
  p->state = UNUSED;
  p->group_leader = 0;
  p->tgid = 0;
  p->ustack = 0;
}
 

 
int
kjoin(uint64 stack_addr)
{
  struct proc *p;
  struct proc *cur_p = myproc();
  int found = 0;
 
  if(cur_p != cur_p->group_leader)
    return -1;
 
  acquire(&wait_lock);
 
  for (;;) {
    found = 0;
    for(p = proc; p < &proc[NPROC]; p++) {
      if (cur_p == p) {
        continue;
      }
      if (p->tgid != cur_p->tgid || p->group_leader == p) {
        continue;
      }
 
      // find sibling thread
      found = 1;
      acquire(&p->lock);
 
      if (p->state != ZOMBIE) {
        release(&p->lock);
        continue;
      }
      
      // 좀비 발견!
      int tid = p->pid;
      uint64 ustack = (uint64)p->ustack;
 
      if(stack_addr != 0 &&  // 스택 넘기기
          copyout(cur_p->mm->pagetable, stack_addr, (char *)&ustack, sizeof(ustack)) < 0) {
        release(&p->lock);
        release(&wait_lock);
        return -1;
      }
 
      // trapframe회수 및 mm_struct도 회수
      freeproc(p);
 
      release(&p->lock);
      release(&wait_lock);
      return tid;
    }
 
    if (found == 0 || killed(cur_p)) { 
      release(&wait_lock);
      return -1;
    }
 
    sleep(cur_p, &wait_lock);
  }
 
}

acquire(&wait_lock);
 
for (;;) {
  // proc table에서 join 대상 thread 찾기
 
  if (zombie thread found)
    freeproc(thread);
 
  if (found == 0 || killed(cur_p))
    return -1;
 
  sleep(cur_p, &wait_lock);
}

int
thread_create(void (*fn)(void *), void *arg, int n_pages)
{
  void *st = malloc(n_pages * PGSIZE);
  int tid = clone(fn, arg, st, n_pages, CLONE_VM);
  if (tid < 0) {
    free(st);
    return -1;
  }
 
  return tid;
}
 
int
thread_join(void)
{
  void *stack;
  int tid = join(&stack);
 
  if (tid < 0) return -1;
 
  free(stack);
  return tid;
}
 

void
kexit(int status)
{
  struct proc *p = myproc();
 
  if(p == initproc)
    panic("init exiting");
 
  // Close all open files.
  for(int fd = 0; fd < NOFILE; fd++){
    if(p->ofile[fd]){
      struct file *f = p->ofile[fd];
      fileclose(f);
      p->ofile[fd] = 0;
    }
  }
 
  begin_op();
  iput(p->cwd);
  end_op();
 
  p->cwd = 0;
 
  kdrainthreads(p);
 
  acquire(&wait_lock);
  // Give any children to init.
  reparent(p);
 
  // Parent might be sleeping in wait().  
  // And the group leader might be sleeping in join().
  wakeup(p->parent);
  
  if (p->group_leader != p) {
    wakeup(p->group_leader);
  } 
 
  acquire(&p->lock);
  p->xstate = status;
  p->state = ZOMBIE;  
 
  release(&wait_lock);
 
  // Jump into the scheduler, never to return.
  sched();
  panic("zombie exit");
}

void
kdrainthreads(struct proc *p)
{
  acquire(&wait_lock);
 
  struct proc *t;
  // 1. sibling까지 다 죽임(kill)
  if(p->group_leader == p) {
    
    for (t = proc; t < &proc[NPROC]; t++) {
      if (t == p) {
        continue;
      }
      if (t->group_leader == p) {
        acquire(&t->lock);
        t->killed = 1;
        if (t->state == SLEEPING) {
          t->state = RUNNABLE;
        }
        release(&t->lock);
      }
    }
    // 2. reaping
    for(;;){
      int have = 0; // scan flag
 
      for(t = proc; t < &proc[NPROC]; t++){
        if(t != p && t->group_leader == p){
          acquire(&t->lock);
          if(t->state == ZOMBIE){
            freeproc(t);
          } else {
            have = 1;
          }
          release(&t->lock);
        }
      }
      if(have == 0)
        break;
      sleep(p, &wait_lock);
    }
  }
 
  release(&wait_lock);
}

int
kkill(int pid)
{
  struct proc *p;
  struct proc *p_gl = 0;
  
  // pid통해 task특정
  for(p = proc; p < &proc[NPROC]; p++){
    acquire(&p->lock);
    if(p->pid == pid){
      p_gl = p->group_leader;
      if(p->state == SLEEPING){
        // Wake process from sleep().
        p->state = RUNNABLE;
      }
    }
    release(&p->lock);
  }
 
  if (p_gl == 0) {
    return -1;  
  } 
 
  int killed_any = 0;
  for(p = proc; p < &proc[NPROC]; p++){
    acquire(&p->lock);
    if(p->group_leader == p_gl){
      p->killed = 1;
      killed_any = 1;
      if(p->state == SLEEPING){
        p->state = RUNNABLE;
      }
    }
    release(&p->lock);
  }
  return killed_any ? 0 : -1;
}

if (p->group_leader != p) {
  return -1;
}
 
beginop();
 
// Open the executable file.
...
// Read the ELF header.
...
// Is this really an ELF file?
...
// Load program into memory.
... 
 
iunlockput(ip);
end_op();
ip = 0;
 
p = myproc();
uint64 oldsz = p->mm->sz;
 
// Allocate some pages at the next page boundary.
// Make the first inaccessible as a stack guard.
// Use the rest as the user stack.
...
// Copy argument strings into new stack, remember their addresses in ustack[].
...
// push a copy of ustack[], the array of argv[] pointers.
...
// a0 and a1 contain arguments to user main(argc, argv)
// argc is returned via the system call return value, which goes in a0.
...
// Save program name for debugging.
...
kdrainthreads(p);
 
// Commit to the user image.
acquire(&p->mm->lock);
oldpagetable = p->mm->pagetable;
p->mm->pagetable = pagetable;
p->mm->sz = sz;
release(&p->mm->lock);
p->trapframe->epc = elf.entry; // initial program counter = ulib.c:start()
p->trapframe->sp = sp; // initial stack pointer
proc_freepagetable(oldpagetable, oldsz);
return argc; // this ends up in a0, the first argument to main(argc, argv)

void
kdrainthreads(struct proc *p)
{
  acquire(&wait_lock);
 
  struct proc *t;
  // 1. sibling까지 다 죽임(kill)
  if(p->group_leader == p) {
    
    for (t = proc; t < &proc[NPROC]; t++) {
      if (t == p) {
        continue;
      }
      if (t->group_leader == p) {
        acquire(&t->lock);
        t->killed = 1;
        if (t->state == SLEEPING) {
          t->state = RUNNABLE;
        }
        release(&t->lock);
      }
    }
    // 2. reaping
    for(;;){
      int have = 0; // scan flag
 
      for(t = proc; t < &proc[NPROC]; t++){
        if(t != p && t->group_leader == p){
          acquire(&t->lock);
          if(t->state == ZOMBIE){
            freeproc(t);
          } else {
            have = 1;
          }
          release(&t->lock);
        }
      }
      if(have == 0)
        break;
      sleep(p, &wait_lock);
    }
  }
 
  release(&wait_lock);
}

// in proc.h
struct proc {
	// ..
	struct proc *group_leader; // group leader (self if not a thread)
	int tgid; // thread group id (== leader->pid)
	uint64 tf_va; // user VA where p->trapframe is mapped
	void *ustack; // user stack base (only set for threads)
}

struct mm_struct {
	pagetable_t pagetable; // page table root
	uint64 sz; // user memory size (bytes)
	int refcount; // number of tasks sharing this mm
	struct spinlock lock;
};

// in trap.c 
void
prepare_return(void) {
	// ..
	w_stvec(trampoline_uservec);
	// .. 
	w_sscratch(p->tf_va); 
	// ..
}
 
// in trampoline.S
uservec:
	csrrw a0, sscratch, a0
	// 이후 save register to trapframe 

TRAMPOLINE
TRAPFRAME                       slot 0, group leader
TRAPFRAME - 1*PGSIZE            slot 1
TRAPFRAME - 2*PGSIZE            slot 2
...
TRAPFRAME - (NTHREAD-1)*PGSIZE  slot 7
USERTOP 아래                    일반 user memory

$ usertests forktest
usertests starting
test forktest: usertrap(): unexpected scause 0xc pid=4
            sepc=0x101010101010101 stval=0x101010101010101
usertrap(): unexpected scause 0xc pid=1
            sepc=0x101010101010101 stval=0x101010101010101
usertrap(): unexpected panscaicu:se 0xc  pid=3
            sepci=0nit exixti1ng
01

static void
freeproc(struct proc *p)
{
  if (p->trapframe) {
    kfree((void *)p->trapframe);
    p->trapframe = 0;
  }
  p->tf_va = 0;
 
  if (p->mm) {
    mm_put(p->mm);
    p->mm = 0;
  }
 
  p->pid = 0;
  p->parent = 0;
  p->name[0] = 0;
  p->chan = 0;
  p->killed = 0;
  p->xstate = 0;
  p->state = UNUSED;
  p->group_leader = 0;
  p->tgid = 0;
  p->ustack = 0;
}
 
void
mm_put(struct mm_struct *mm)
{
	struct proc *p = myproc();
	acquire(&mm->lock);
	mm->refcount--;
	uvmunmap(mm->pagetable, p->tf_va, 1, 1);
	kfree(p->trapframe); // trapframe free 책임이 중복
	release(&mm->lock);
}

// kernel/proc.c 
 
for(p = proc; p < &proc[NPROC]; p++) { 
	char *pa = kalloc(); 
	uint64 va = KSTACK((int) (p - proc)); 
	kvmmap(kpgtbl, va, (uint64)pa, PGSIZE, PTE_R | PTE_W); 
}

// kernel/memlayout.h
#define KSTACK(p) (TRAMPOLINE - ((p)+1)* 2*PGSIZE)

// kernel/proc.c  
p->kstack = KSTACK((int) (p - proc));

// kernel/proc.c  
p->context.ra = (uint64)forkret; 
p->context.sp = p->kstack + PGSIZE;

private_value = 111;
pid = fork();
if(pid == 0){
  private_value = 222;
  exit(WAIT_STATUS);
}
 
got = wait(&status);
CHECK_EQ(private_value, 111, "process clone unexpectedly shared memory");

shared_value = 0;
tid = clone_one(shared_worker, &arg, &stack);
CHECK(tid > 0, "clone(CLONE_VM) failed");
join_one(tid, stack);
CHECK_EQ(shared_value, 42, "thread did not share address space");
CHECK_EQ(wait(0), -1, "wait should not reap a joined thread");

tids[i] = clone(slot_worker, &args[i], stacks[i], 1, CLONE_VM);
 
int joined_tid = join(&joined_stack);
CHECK(joined_tid > 0, "join failed before all threads were collected");
CHECK(known_stack_index(joined_stack, stacks, seen, NJOIN) >= 0,
      "join returned an unknown or duplicate stack");

pid = fork();
if(pid == 0)
  exit(WAIT_STATUS);
 
got = wait(&status);
CHECK_EQ(got, pid, "wait returned the wrong child pid");
CHECK_EQ(status, WAIT_STATUS, "wait returned the wrong exit status");
CHECK_EQ(wait(0), -1, "wait with no children should fail");

tid = clone_one(shared_worker, &arg, &stack);
while(shared_value == 0)
  pause(1);
 
CHECK_EQ(wait(0), -1, "wait should not reap a sibling thread");
join_one(tid, stack);

tid = clone_one(join_from_thread_worker, 0, &stack);
join_one(tid, stack);
CHECK_EQ(join_from_thread_result, -1, "non-leader thread should not join");

join_from_thread_result = join(0);
exit(0);

arg.oldbrk = sbrk(0);
tid = clone_one(sbrk_worker, &arg, &stack);
join_one(tid, stack);
 
CHECK_PTR_EQ(arg.allocated, arg.oldbrk, "thread sbrk used the wrong break");
CHECK_EQ(arg.allocated[0], 'T', "parent could not read first byte");
CHECK_PTR_EQ(sbrk(0), arg.oldbrk + PGSIZE,
             "parent did not observe shared break growth");

if(pid == 0){
  tid = clone_one(sleeper_worker, (void *)&sleeper_ready, &stack);
  while(sleeper_ready == 0)
    pause(1);
 
  exit(LEADER_EXIT_STATUS);
}
 
got = wait(&status);
CHECK_EQ(got, pid, "wait returned the wrong child pid");
CHECK_EQ(status, LEADER_EXIT_STATUS, "leader exit status was not preserved");

CHECK_EQ(kill(pid), 0, "kill returned failure for live child");
got = wait(&status);
CHECK_EQ(got, pid, "wait returned the wrong killed child pid");
CHECK_EQ(status, -1, "killed child should report status -1");
CHECK_EQ(kill(pid), -1, "kill should fail after child is reaped");

tid = clone_one(sleeper_worker, (void *)&sleeper_ready, &stack);
CHECK_EQ(write(fds[1], &tid, sizeof(tid)), sizeof(tid),
         "child failed to report thread tid");

CHECK_EQ(kill(tid), 0, "kill by thread tid failed");
got = wait(&status);
CHECK_EQ(got, pid, "wait should reap the killed group leader");
CHECK_EQ(status, -1, "killed thread group should report status -1");

tid = thread_create(wrapper_worker, &arg, 1);
CHECK(tid > 0, "thread_create failed");
joined_tid = thread_join();
CHECK_EQ(joined_tid, tid, "thread_join returned the wrong tid");
CHECK_EQ(wrapper_value, 777, "wrapper thread did not share memory");