SmartAudio/lichee/linux-4.9/drivers/gator/gator_trace_sched.c

/**
 * Copyright (C) ARM Limited 2010-2016. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 */

#include <trace/events/sched.h>
#include <trace/events/task.h>

#include "gator.h"

#define TASK_MAP_ENTRIES        1024    /* must be power of 2 */
#define TASK_MAX_COLLISIONS     2

enum {
    STATE_WAIT_ON_OTHER = 0,
    STATE_CONTENTION,
    STATE_WAIT_ON_IO,
    CPU_WAIT_TOTAL
};

static DEFINE_PER_CPU(uint64_t *, taskname_keys);
static DEFINE_PER_CPU(int, collecting);

/* this array is never read as the cpu charts are derived
 * counters the files are needed, nonetheless, to show that these
 * counters are available
 */
static const char *sched_wait_event_names[] = {
    "Linux_cpu_wait_contention",
    "Linux_cpu_wait_io",
};
static ulong sched_wait_enabled[ARRAY_SIZE(sched_wait_event_names)];
static ulong sched_wait_keys[ARRAY_SIZE(sched_wait_event_names)];

static const char *sched_activity_event_names[] = {
    "system",
    "user",
};
static ulong sched_activity_enabled[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT];
static ulong sched_activity_keys[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT];

static int sched_trace_create_files(struct super_block *sb, struct dentry *root)
{
    struct dentry *dir;
    int i;
    int j;
    char buf[40];

    for (i = 0; i < ARRAY_SIZE(sched_wait_event_names); ++i) {
        dir = gatorfs_mkdir(sb, root, sched_wait_event_names[i]);
        if (!dir)
            return -1;
        gatorfs_create_ulong(sb, dir, "enabled", &sched_wait_enabled[i]);
        gatorfs_create_ro_ulong(sb, dir, "key", &sched_wait_keys[i]);
    }

    for (i = 0; i < ARRAY_SIZE(sched_activity_event_names); ++i) {
        for (j = 0; j < gator_cluster_count; j++) {
            snprintf(buf, sizeof(buf), "%s_%s", gator_clusters[j]->pmnc_name, sched_activity_event_names[i]);
            dir = gatorfs_mkdir(sb, root, buf);
            if (!dir)
                return -1;
            gatorfs_create_ulong(sb, dir, "enabled", &sched_activity_enabled[i][j]);
            gatorfs_create_ro_ulong(sb, dir, "key", &sched_activity_keys[i][j]);
        }
    }

    return 0;
}

static void emit_pid_name(const char *comm, struct task_struct *task)
{
    bool found = false;
    char taskcomm[TASK_COMM_LEN + 3];
    unsigned long x, cpu = get_physical_cpu();
    uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
    uint64_t value;

    value = gator_chksum_crc32(comm);
    value = (value << 32) | (uint32_t)task->pid;

    /* determine if the thread name was emitted already */
    for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
        if (keys[x] == value) {
            found = true;
            break;
        }
    }

    if (!found) {
        /* shift values, new value always in front */
        uint64_t oldv, newv = value;

        for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
            oldv = keys[x];
            keys[x] = newv;
            newv = oldv;
        }

        /* emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions */
        if (strlcpy(taskcomm, comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
            /* append ellipses if comm has length of TASK_COMM_LEN - 1 */
            strcat(taskcomm, "...");
        }

        marshal_thread_name(task->pid, taskcomm);
    }
}

static void collect_counters(u64 time, struct task_struct *task, bool sched_switch)
{
    int *buffer, len, cpu = get_physical_cpu();
    long long *buffer64;
    struct gator_interface *gi;

    if (marshal_event_header(time)) {
        list_for_each_entry(gi, &gator_events, list) {
            if (gi->read) {
                len = gi->read(&buffer, sched_switch);
                if (len < 0)
                    pr_err("gator: read failed for %s\n", gi->name);
                marshal_event(len, buffer);
            } else if (gi->read64) {
                len = gi->read64(&buffer64, sched_switch);
                if (len < 0)
                    pr_err("gator: read64 failed for %s\n", gi->name);
                marshal_event64(len, buffer64);
            }
            if (gi->read_proc && task != NULL) {
                len = gi->read_proc(&buffer64, task);
                if (len < 0)
                    pr_err("gator: read_proc failed for %s\n", gi->name);
                marshal_event64(len, buffer64);
            }
        }
        if (cpu == 0)
            gator_emit_perf_time(time);
        /* Only check after writing all counters so that time and corresponding counters appear in the same frame */
        buffer_check(cpu, BLOCK_COUNTER_BUF, time);

        /* Commit buffers on timeout */
        if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) {
            static const int buftypes[] = { NAME_BUF, COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF, ACTIVITY_BUF };
            int i;

            for (i = 0; i < ARRAY_SIZE(buftypes); ++i)
                gator_commit_buffer(cpu, buftypes[i], time);

            /* spinlocks are noops on uniprocessor machines and mutexes do
             * not work in sched_switch context in RT-Preempt full, so
             * disable proactive flushing of the annotate frame on
             * uniprocessor machines.
             */
#ifdef CONFIG_SMP
            /* Try to preemptively flush the annotate buffer to reduce the chance of the buffer being full */
            if (on_primary_core() && spin_trylock(&annotate_lock)) {
                gator_commit_buffer(0, ANNOTATE_BUF, time);
                spin_unlock(&annotate_lock);
            }
#endif
        }
    }
}

/* special case used during a suspend of the system */
static void trace_sched_insert_idle(void)
{
    marshal_sched_trace_switch(0, 0);
}

static void gator_trace_emit_link(struct task_struct *p)
{
    int cookie;
    int cpu = get_physical_cpu();

    cookie = get_exec_cookie(cpu, p);
    emit_pid_name(p->comm, p);

    marshal_link(cookie, p->tgid, p->pid);
}

GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child))
{
    gator_trace_emit_link(child);
}

GATOR_DEFINE_PROBE(sched_process_exec, TP_PROTO(struct task_struct *p, pid_t old_pid, struct linux_binprm *bprm))
{
    gator_trace_emit_link(p);
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct *task, char *comm))
#else
GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct *task, const char *comm))
#endif
{
    emit_pid_name(comm, task);
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
#else
GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(bool preempt, struct task_struct *prev, struct task_struct *next))
#endif
{
    int state;
    int cpu = get_physical_cpu();

    per_cpu(in_scheduler_context, cpu) = true;

    /* do as much work as possible before disabling interrupts */
    if (prev->state == TASK_RUNNING)
        state = STATE_CONTENTION;
    else if (prev->in_iowait)
        state = STATE_WAIT_ON_IO;
    else
        state = STATE_WAIT_ON_OTHER;

    per_cpu(collecting, cpu) = 1;
    collect_counters(gator_get_time(), prev, true);
    per_cpu(collecting, cpu) = 0;

    marshal_sched_trace_switch(next->pid, state);

    per_cpu(in_scheduler_context, cpu) = false;
}

GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
{
    marshal_sched_trace_exit(p->tgid, p->pid);
}

static void do_nothing(void *info)
{
    /* Intentionally do nothing */
    (void)info;
}

static int register_scheduler_tracepoints(void)
{
    /* register tracepoints */
    if (GATOR_REGISTER_TRACE(sched_process_fork))
        goto fail_sched_process_fork;
    if (GATOR_REGISTER_TRACE(sched_process_exec))
        goto fail_sched_process_exec;
    if (GATOR_REGISTER_TRACE(task_rename))
        goto fail_task_rename;
    if (GATOR_REGISTER_TRACE(sched_switch))
        goto fail_sched_switch;
    if (GATOR_REGISTER_TRACE(sched_process_free))
        goto fail_sched_process_free;
    pr_debug("gator: registered tracepoints\n");

    /* Now that the scheduler tracepoint is registered, force a context
     * switch on all cpus to capture what is currently running.
     */
    on_each_cpu(do_nothing, NULL, 0);

    return 0;

    /* unregister tracepoints on error */
fail_sched_process_free:
    GATOR_UNREGISTER_TRACE(sched_switch);
fail_sched_switch:
    GATOR_UNREGISTER_TRACE(task_rename);
fail_task_rename:
    GATOR_UNREGISTER_TRACE(sched_process_exec);
fail_sched_process_exec:
    GATOR_UNREGISTER_TRACE(sched_process_fork);
fail_sched_process_fork:
    pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");

    return -1;
}

static void unregister_scheduler_tracepoints(void)
{
    GATOR_UNREGISTER_TRACE(sched_process_fork);
    GATOR_UNREGISTER_TRACE(sched_process_exec);
    GATOR_UNREGISTER_TRACE(task_rename);
    GATOR_UNREGISTER_TRACE(sched_switch);
    GATOR_UNREGISTER_TRACE(sched_process_free);
    pr_debug("gator: unregistered tracepoints\n");
}

static void gator_trace_sched_stop(void)
{
    int cpu;

    unregister_scheduler_tracepoints();

    for_each_present_cpu(cpu) {
        kfree(per_cpu(taskname_keys, cpu));
    }
}

static int gator_trace_sched_start(void)
{
    int cpu, size;
    int ret;

    for_each_present_cpu(cpu) {
        size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
        per_cpu(taskname_keys, cpu) = kmalloc(size, GFP_KERNEL);
        if (!per_cpu(taskname_keys, cpu))
            return -1;
        memset(per_cpu(taskname_keys, cpu), 0, size);
    }

    ret = register_scheduler_tracepoints();

    return ret;
}

static void gator_trace_sched_offline(void)
{
    trace_sched_insert_idle();
}

static void gator_trace_sched_init(void)
{
    int i;
    int j;

    for (i = 0; i < ARRAY_SIZE(sched_wait_enabled); i++) {
        sched_wait_enabled[i] = 0;
        sched_wait_keys[i] = gator_events_get_key();
    }

    for (i = 0; i < ARRAY_SIZE(sched_activity_enabled); i++) {
        for (j = 0; j < gator_cluster_count; j++) {
            sched_activity_enabled[i][j] = 0;
            sched_activity_keys[i][j] = gator_events_get_key();
        }
    }
}