*
* Derived from the read-mod example from relay-examples by Tom Zanussi.
*/
+#define DEBUG 1
+
#include <linux/module.h>
-#include <linux/relay.h>
#include <linux/debugfs.h>
-#include <linux/proc_fs.h>
+#include <linux/uaccess.h>
#include <asm/io.h>
#include <linux/version.h>
#include <linux/kallsyms.h>
#include <asm/atomic.h>
#include <linux/percpu.h>
-#include "kmmio.h"
#include "pf_in.h"
-/* This app's relay channel files will appear in /debug/mmio-trace */
-#define APP_DIR "mmio-trace"
-/* the marker injection file in /proc */
-#define MARKER_FILE "mmio-marker"
+#define NAME "mmiotrace: "
-#define MODULE_NAME "mmiotrace"
+/* This app's relay channel files will appear in /debug/mmio-trace */
+static const char APP_DIR[] = "mmio-trace";
+/* the marker injection file in /debug/APP_DIR */
+static const char MARKER_FILE[] = "mmio-marker";
struct trap_reason {
unsigned long addr;
int active_traces;
};
+struct remap_trace {
+ struct list_head list;
+ struct kmmio_probe probe;
+ unsigned long phys;
+ unsigned long id;
+};
+
+static const size_t subbuf_size = 256*1024;
+
/* Accessed per-cpu. */
static DEFINE_PER_CPU(struct trap_reason, pf_reason);
static DEFINE_PER_CPU(struct mm_io_header_rw, cpu_trace);
+#if 0 /* XXX: no way gather this info anymore */
/* Access to this is not per-cpu. */
static DEFINE_PER_CPU(atomic_t, dropped);
+#endif
-static struct file_operations mmio_fops = {
- .owner = THIS_MODULE,
-};
-
-static const size_t subbuf_size = 256*1024;
-static struct rchan *chan;
static struct dentry *dir;
-static struct proc_dir_entry *proc_marker_file;
+static struct dentry *marker_file;
+
+static DEFINE_MUTEX(mmiotrace_mutex);
+static DEFINE_SPINLOCK(trace_lock);
+static atomic_t mmiotrace_enabled;
+static LIST_HEAD(trace_list); /* struct remap_trace */
+
+/*
+ * Locking in this file:
+ * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
+ * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
+ * and trace_lock.
+ * - Routines depending on is_enabled() must take trace_lock.
+ * - trace_list users must hold trace_lock.
+ * - is_enabled() guarantees that chan is valid.
+ * - pre/post callbacks assume the effect of is_enabled() being true.
+ */
/* module parameters */
-static unsigned int n_subbufs = 32*4;
-static unsigned long filter_offset;
-static int nommiotrace;
-static int ISA_trace;
-static int trace_pc;
+static unsigned int n_subbufs = 32*4;
+static unsigned long filter_offset;
+static int nommiotrace;
+static int ISA_trace;
+static int trace_pc;
module_param(n_subbufs, uint, 0);
module_param(filter_offset, ulong, 0);
MODULE_PARM_DESC(ISA_trace, "Do not exclude the low ISA range.");
MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
-static void record_timestamp(struct mm_io_header *header)
+static bool is_enabled(void)
{
- struct timespec now;
-
- getnstimeofday(&now);
- header->sec = now.tv_sec;
- header->nsec = now.tv_nsec;
+ return atomic_read(&mmiotrace_enabled);
}
/*
- * Write callback for the /proc entry:
+ * Write callback for the debugfs entry:
* Read a marker and write it to the mmio trace log
*/
-static int write_marker(struct file *file, const char __user *buffer,
- unsigned long count, void *data)
+static ssize_t write_marker(struct file *file, const char __user *buffer,
+ size_t count, loff_t *ppos)
{
char *event = NULL;
struct mm_io_header *headp;
- int len = (count > 65535) ? 65535 : count;
+ ssize_t len = (count > 65535) ? 65535 : count;
event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
if (!event)
headp = (struct mm_io_header *)event;
headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
headp->data_len = len;
- record_timestamp(headp);
if (copy_from_user(event + sizeof(*headp), buffer, len)) {
kfree(event);
return -EFAULT;
}
- relay_write(chan, event, sizeof(*headp) + len);
+ spin_lock_irq(&trace_lock);
+#if 0 /* XXX: convert this to use tracing */
+ if (is_enabled())
+ relay_write(chan, event, sizeof(*headp) + len);
+ else
+#endif
+ len = -EINVAL;
+ spin_unlock_irq(&trace_lock);
kfree(event);
return len;
}
pte_t *pte = lookup_address(address, &level);
if (!pte) {
- printk(KERN_ERR "Error in %s: no pte for page 0x%08lx\n",
- __FUNCTION__, address);
+ pr_err(NAME "Error in %s: no pte for page 0x%08lx\n",
+ __func__, address);
return;
}
if (level == PG_LEVEL_2M) {
- printk(KERN_EMERG MODULE_NAME ": 4MB pages are not "
- "currently supported: %lx\n",
- address);
+ pr_emerg(NAME "4MB pages are not currently supported: "
+ "0x%08lx\n", address);
BUG();
}
- printk(KERN_DEBUG MODULE_NAME ": pte for 0x%lx: 0x%lx 0x%lx\n",
- address, pte_val(*pte),
- pte_val(*pte) & _PAGE_PRESENT);
+ pr_info(NAME "pte for 0x%lx: 0x%lx 0x%lx\n", address, pte_val(*pte),
+ pte_val(*pte) & _PAGE_PRESENT);
}
/*
static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
{
const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
- printk(KERN_EMERG MODULE_NAME ": unexpected fault for address: %lx, "
- "last fault for address: %lx\n",
+ pr_emerg(NAME "unexpected fault for address: 0x%08lx, "
+ "last fault for address: 0x%08lx\n",
addr, my_reason->addr);
print_pte(addr);
+ print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
+ print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
#ifdef __i386__
- print_symbol(KERN_EMERG "faulting EIP is at %s\n", regs->ip);
- print_symbol(KERN_EMERG "last faulting EIP was at %s\n",
- my_reason->ip);
- printk(KERN_EMERG
- "eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n",
+ pr_emerg("eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n",
regs->ax, regs->bx, regs->cx, regs->dx);
- printk(KERN_EMERG
- "esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n",
+ pr_emerg("esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n",
regs->si, regs->di, regs->bp, regs->sp);
#else
- print_symbol(KERN_EMERG "faulting RIP is at %s\n", regs->ip);
- print_symbol(KERN_EMERG "last faulting RIP was at %s\n",
- my_reason->ip);
- printk(KERN_EMERG "rax: %016lx rcx: %016lx rdx: %016lx\n",
+ pr_emerg("rax: %016lx rcx: %016lx rdx: %016lx\n",
regs->ax, regs->cx, regs->dx);
- printk(KERN_EMERG "rsi: %016lx rdi: %016lx "
- "rbp: %016lx rsp: %016lx\n",
+ pr_emerg("rsi: %016lx rdi: %016lx rbp: %016lx rsp: %016lx\n",
regs->si, regs->di, regs->bp, regs->sp);
#endif
put_cpu_var(pf_reason);
my_trace->header.pid = 0;
my_trace->header.data_len = sizeof(struct mm_io_rw);
my_trace->rw.address = addr;
+ /*
+ * struct remap_trace *trace = p->user_data;
+ * phys = addr - trace->probe.addr + trace->phys;
+ */
/*
* Only record the program counter when requested.
else
my_trace->rw.pc = 0;
- record_timestamp(&my_trace->header);
+ /*
+ * XXX: the timestamp recorded will be *after* the tracing has been
+ * done, not at the time we hit the instruction. SMP implications
+ * on event ordering?
+ */
switch (type) {
case REG_READ:
/* this should always return the active_trace count to 0 */
my_reason->active_traces--;
if (my_reason->active_traces) {
- printk(KERN_EMERG MODULE_NAME ": unexpected post handler");
+ pr_emerg(NAME "unexpected post handler");
BUG();
}
default:
break;
}
- relay_write(chan, my_trace, sizeof(*my_trace));
+
+ /*
+ * XXX: Several required values are ignored:
+ * - mapping id
+ * - program counter
+ * Also the address should be physical, not virtual.
+ */
+ mmio_trace_record(my_trace->header.type, my_trace->rw.address,
+ my_trace->rw.value);
put_cpu_var(cpu_trace);
put_cpu_var(pf_reason);
}
-/*
- * subbuf_start() relay callback.
- *
- * Defined so that we know when events are dropped due to the buffer-full
- * condition.
- */
-static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
- void *prev_subbuf, size_t prev_padding)
-{
- unsigned int cpu = buf->cpu;
- atomic_t *drop = &per_cpu(dropped, cpu);
- int count;
- if (relay_buf_full(buf)) {
- if (atomic_inc_return(drop) == 1) {
- printk(KERN_ERR MODULE_NAME ": cpu %d buffer full!\n",
- cpu);
- }
- return 0;
- } else if ((count = atomic_read(drop))) {
- printk(KERN_ERR MODULE_NAME
- ": cpu %d buffer no longer full, "
- "missed %d events.\n",
- cpu, count);
- atomic_sub(count, drop);
- }
-
- return 1;
-}
-
-/* file_create() callback. Creates relay file in debugfs. */
-static struct dentry *create_buf_file_handler(const char *filename,
- struct dentry *parent,
- int mode,
- struct rchan_buf *buf,
- int *is_global)
-{
- struct dentry *buf_file;
-
- mmio_fops.read = relay_file_operations.read;
- mmio_fops.open = relay_file_operations.open;
- mmio_fops.poll = relay_file_operations.poll;
- mmio_fops.mmap = relay_file_operations.mmap;
- mmio_fops.release = relay_file_operations.release;
- mmio_fops.splice_read = relay_file_operations.splice_read;
-
- buf_file = debugfs_create_file(filename, mode, parent, buf,
- &mmio_fops);
-
- return buf_file;
-}
-
-/* file_remove() default callback. Removes relay file in debugfs. */
-static int remove_buf_file_handler(struct dentry *dentry)
-{
- debugfs_remove(dentry);
- return 0;
-}
-
-static struct rchan_callbacks relay_callbacks = {
- .subbuf_start = subbuf_start_handler,
- .create_buf_file = create_buf_file_handler,
- .remove_buf_file = remove_buf_file_handler,
-};
-
-/*
- * create_channel - creates channel /debug/APP_DIR/cpuXXX
- * Returns channel on success, NULL otherwise
- */
-static struct rchan *create_channel(unsigned size, unsigned n)
-{
- return relay_open("cpu", dir, size, n, &relay_callbacks, NULL);
-}
-
-/* destroy_channel - destroys channel /debug/APP_DIR/cpuXXX */
-static void destroy_channel(void)
-{
- if (chan) {
- relay_close(chan);
- chan = NULL;
- }
-}
-
-struct remap_trace {
- struct list_head list;
- struct kmmio_probe probe;
-};
-static LIST_HEAD(trace_list);
-static DEFINE_SPINLOCK(trace_list_lock);
-
-static void do_ioremap_trace_core(unsigned long offset, unsigned long size,
+static void ioremap_trace_core(unsigned long offset, unsigned long size,
void __iomem *addr)
{
+ static atomic_t next_id;
struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
struct mm_io_header_map event = {
.header = {
.pc = 0
}
};
- record_timestamp(&event.header);
+
+ if (!trace) {
+ pr_err(NAME "kmalloc failed in ioremap\n");
+ return;
+ }
*trace = (struct remap_trace) {
.probe = {
.len = size,
.pre_handler = pre,
.post_handler = post,
- }
+ .user_data = trace
+ },
+ .phys = offset,
+ .id = atomic_inc_return(&next_id)
};
- relay_write(chan, &event, sizeof(event));
- spin_lock(&trace_list_lock);
+ spin_lock_irq(&trace_lock);
+ if (!is_enabled())
+ goto not_enabled;
+
+ /*
+ * XXX: Insufficient data recorded!
+ */
+ mmio_trace_record(event.header.type, event.map.addr, event.map.len);
list_add_tail(&trace->list, &trace_list);
- spin_unlock(&trace_list_lock);
if (!nommiotrace)
register_kmmio_probe(&trace->probe);
+
+not_enabled:
+ spin_unlock_irq(&trace_lock);
}
-static void ioremap_trace_core(unsigned long offset, unsigned long size,
- void __iomem *addr)
+void
+mmiotrace_ioremap(unsigned long offset, unsigned long size, void __iomem *addr)
{
- if ((filter_offset) && (offset != filter_offset))
+ if (!is_enabled()) /* recheck and proper locking in *_core() */
return;
- /* Don't trace the low PCI/ISA area, it's always mapped.. */
- if (!ISA_trace && (offset < ISA_END_ADDRESS) &&
- (offset + size > ISA_START_ADDRESS)) {
- printk(KERN_NOTICE MODULE_NAME ": Ignoring map of low "
- "PCI/ISA area (0x%lx-0x%lx)\n",
- offset, offset + size);
+ pr_debug(NAME "ioremap_*(0x%lx, 0x%lx) = %p\n", offset, size, addr);
+ if ((filter_offset) && (offset != filter_offset))
return;
- }
- do_ioremap_trace_core(offset, size, addr);
+ ioremap_trace_core(offset, size, addr);
}
-void __iomem *ioremap_cache_trace(unsigned long offset, unsigned long size)
-{
- void __iomem *p = ioremap_cache(offset, size);
- printk(KERN_DEBUG MODULE_NAME ": ioremap_cache(0x%lx, 0x%lx) = %p\n",
- offset, size, p);
- ioremap_trace_core(offset, size, p);
- return p;
-}
-EXPORT_SYMBOL(ioremap_cache_trace);
-
-void __iomem *ioremap_nocache_trace(unsigned long offset, unsigned long size)
-{
- void __iomem *p = ioremap_nocache(offset, size);
- printk(KERN_DEBUG MODULE_NAME ": ioremap_nocache(0x%lx, 0x%lx) = %p\n",
- offset, size, p);
- ioremap_trace_core(offset, size, p);
- return p;
-}
-EXPORT_SYMBOL(ioremap_nocache_trace);
-
-void iounmap_trace(volatile void __iomem *addr)
+static void iounmap_trace_core(volatile void __iomem *addr)
{
struct mm_io_header_map event = {
.header = {
};
struct remap_trace *trace;
struct remap_trace *tmp;
- printk(KERN_DEBUG MODULE_NAME ": Unmapping %p.\n", addr);
- record_timestamp(&event.header);
+ struct remap_trace *found_trace = NULL;
+
+ pr_debug(NAME "Unmapping %p.\n", addr);
+
+ spin_lock_irq(&trace_lock);
+ if (!is_enabled())
+ goto not_enabled;
- spin_lock(&trace_list_lock);
list_for_each_entry_safe(trace, tmp, &trace_list, list) {
if ((unsigned long)addr == trace->probe.addr) {
if (!nommiotrace)
unregister_kmmio_probe(&trace->probe);
list_del(&trace->list);
- kfree(trace);
+ found_trace = trace;
break;
}
}
- spin_unlock(&trace_list_lock);
- relay_write(chan, &event, sizeof(event));
- iounmap(addr);
+ mmio_trace_record(event.header.type, event.map.addr,
+ found_trace ? found_trace->id : -1);
+
+not_enabled:
+ spin_unlock_irq(&trace_lock);
+ if (found_trace) {
+ synchronize_rcu(); /* unregister_kmmio_probe() requirement */
+ kfree(found_trace);
+ }
+}
+
+void mmiotrace_iounmap(volatile void __iomem *addr)
+{
+ might_sleep();
+ if (is_enabled()) /* recheck and proper locking in *_core() */
+ iounmap_trace_core(addr);
}
-EXPORT_SYMBOL(iounmap_trace);
static void clear_trace_list(void)
{
struct remap_trace *trace;
struct remap_trace *tmp;
- spin_lock(&trace_list_lock);
- list_for_each_entry_safe(trace, tmp, &trace_list, list) {
- printk(KERN_WARNING MODULE_NAME ": purging non-iounmapped "
+ /*
+ * No locking required, because the caller ensures we are in a
+ * critical section via mutex, and is_enabled() is false,
+ * i.e. nothing can traverse or modify this list.
+ * Caller also ensures is_enabled() cannot change.
+ */
+ list_for_each_entry(trace, &trace_list, list) {
+ pr_notice(NAME "purging non-iounmapped "
"trace @0x%08lx, size 0x%lx.\n",
trace->probe.addr, trace->probe.len);
if (!nommiotrace)
unregister_kmmio_probe(&trace->probe);
+ }
+ synchronize_rcu(); /* unregister_kmmio_probe() requirement */
+
+ list_for_each_entry_safe(trace, tmp, &trace_list, list) {
list_del(&trace->list);
kfree(trace);
- break;
}
- spin_unlock(&trace_list_lock);
}
-static int __init init(void)
-{
- if (n_subbufs < 2)
- return -EINVAL;
-
- dir = debugfs_create_dir(APP_DIR, NULL);
- if (!dir) {
- printk(KERN_ERR MODULE_NAME
- ": Couldn't create relay app directory.\n");
- return -ENOMEM;
- }
+static struct file_operations fops_marker = {
+ .owner = THIS_MODULE,
+ .write = write_marker
+};
- chan = create_channel(subbuf_size, n_subbufs);
- if (!chan) {
- debugfs_remove(dir);
- printk(KERN_ERR MODULE_NAME
- ": relay app channel creation failed\n");
- return -ENOMEM;
- }
+void enable_mmiotrace(void)
+{
+ mutex_lock(&mmiotrace_mutex);
+ if (is_enabled())
+ goto out;
- init_kmmio();
+ reference_kmmio();
- proc_marker_file = create_proc_entry(MARKER_FILE, 0, NULL);
- if (proc_marker_file)
- proc_marker_file->write_proc = write_marker;
+#if 0 /* XXX: tracing does not support text entries */
+ marker_file = debugfs_create_file("marker", 0660, dir, NULL,
+ &fops_marker);
+#endif
+ if (!marker_file)
+ pr_err(NAME "marker file creation failed.\n");
- printk(KERN_DEBUG MODULE_NAME ": loaded.\n");
if (nommiotrace)
- printk(KERN_DEBUG MODULE_NAME ": MMIO tracing disabled.\n");
+ pr_info(NAME "MMIO tracing disabled.\n");
if (ISA_trace)
- printk(KERN_WARNING MODULE_NAME
- ": Warning! low ISA range will be traced.\n");
- return 0;
+ pr_warning(NAME "Warning! low ISA range will be traced.\n");
+ spin_lock_irq(&trace_lock);
+ atomic_inc(&mmiotrace_enabled);
+ spin_unlock_irq(&trace_lock);
+ pr_info(NAME "enabled.\n");
+out:
+ mutex_unlock(&mmiotrace_mutex);
}
-static void __exit cleanup(void)
+void disable_mmiotrace(void)
{
- printk(KERN_DEBUG MODULE_NAME ": unload...\n");
- clear_trace_list();
- cleanup_kmmio();
- remove_proc_entry(MARKER_FILE, NULL);
- destroy_channel();
- if (dir)
- debugfs_remove(dir);
+ mutex_lock(&mmiotrace_mutex);
+ if (!is_enabled())
+ goto out;
+
+ spin_lock_irq(&trace_lock);
+ atomic_dec(&mmiotrace_enabled);
+ BUG_ON(is_enabled());
+ spin_unlock_irq(&trace_lock);
+
+ clear_trace_list(); /* guarantees: no more kmmio callbacks */
+ unreference_kmmio();
+ if (marker_file) {
+ debugfs_remove(marker_file);
+ marker_file = NULL;
+ }
+
+ pr_info(NAME "disabled.\n");
+out:
+ mutex_unlock(&mmiotrace_mutex);
}
-module_init(init);
-module_exit(cleanup);
-MODULE_LICENSE("GPL");
+int __init init_mmiotrace(void)
+{
+ pr_debug(NAME "load...\n");
+ if (n_subbufs < 2)
+ return -EINVAL;
+
+ dir = debugfs_create_dir(APP_DIR, NULL);
+ if (!dir) {
+ pr_err(NAME "Couldn't create relay app directory.\n");
+ return -ENOMEM;
+ }
+ return 0;
+}