From bab62e1974700089020daa22c3c68bf4a06717e2 Mon Sep 17 00:00:00 2001 From: rubberhead Date: Mon, 12 Feb 2024 20:23:59 +0000 Subject: [PATCH] Edited userspace impl, cleaned up my_shmem impl --- src/aarch64-linux-flush-dcache/my_shmem.c | 32 ++++++++------- .../userspace/writer_null.c | 29 +++++++++++-- src/rdma-test/.vscode-ctags | 38 +++++++++++++++++ src/rdma-test/main.c | 41 +++++++++++++++++++ 4 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 src/rdma-test/.vscode-ctags create mode 100644 src/rdma-test/main.c diff --git a/src/aarch64-linux-flush-dcache/my_shmem.c b/src/aarch64-linux-flush-dcache/my_shmem.c index 3c0e5ee..9430a24 100644 --- a/src/aarch64-linux-flush-dcache/my_shmem.c +++ b/src/aarch64-linux-flush-dcache/my_shmem.c @@ -55,14 +55,6 @@ static void my_shmem_vmops_close(struct vm_area_struct *vma) { pr_info("[%s] Entered.\n", __func__); - /* [?] - * For some reason, `get_user_pages` on vma always fails with EFAULT. - * I feel like this should only be the case if `close` is called only - * after PT entries are removed. It looks like so (re: exit_mmap:3322) - * but is this the case? - * - * Or maybe it's `unmap_vmas` which invalidates everything in MMU? - */ size_t nr_pages_of_vma = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; size_t nr_pages_offset = vma->vm_pgoff; struct my_shmem_page *entry; @@ -105,18 +97,21 @@ static vm_fault_t my_shmem_vmops_fault(struct vm_fault *vmf) pr_info("[%s] vm_fault @ 0x%lx (real address 0x%lx).\n", __func__, vmf->address, vmf->real_address); + vm_fault_t ret = 0; struct vm_area_struct *vma_of_vmf = vmf->vma; - ulong nr_pages_from_vm_start = + ulong vma_pg_offset = vma_of_vmf->vm_pgoff; + ulong vmf_pg_offset_from_vma_bgn = (vmf->address - vma_of_vmf->vm_start) >> PAGE_SHIFT; + ulong pg_offset = vma_pg_offset + vmf_pg_offset_from_vma_bgn; phys_addr_t _dbg_phys_of_page; struct page *last_pg; mutex_lock(&my_shmem_pages_mtx); size_t my_shmem_page_count = list_count_nodes(&my_shmem_pages); - BUG_ON(nr_pages_from_vm_start < my_shmem_page_count); + BUG_ON(pg_offset < my_shmem_page_count); /* Allocate the new page(s) */ - ulong nr_pages_to_alloc = nr_pages_from_vm_start - my_shmem_page_count + 1; + ulong nr_pages_to_alloc = pg_offset - my_shmem_page_count + 1; pr_info("[%s] Page count %ld, offset %ld -- allocating %ld more...\n", __func__, my_shmem_page_count, vmf->pgoff, nr_pages_to_alloc); for (; nr_pages_to_alloc > 0; nr_pages_to_alloc--) @@ -138,11 +133,12 @@ static vm_fault_t my_shmem_vmops_fault(struct vm_fault *vmf) pr_info("[%s] Allocated pfn: %ld, kernel vaddr: %px.\n", __func__, page_to_pfn(curr_pg), page_to_virt(curr_pg)); get_page(curr_pg); // For base page refcount + + // Populate page handle new_page->page = curr_pg; - // List maintenance: add and incr. size + // Add page handle to list list_add(&new_page->list, &my_shmem_pages); - // my_shmem_page_count++; // Fill in last_pg for final return from page fault handler last_pg = curr_pg; @@ -150,6 +146,14 @@ static vm_fault_t my_shmem_vmops_fault(struct vm_fault *vmf) // Fill in vmf's page for return get_page(last_pg); vmf->page = last_pg; + // ret = vmf_insert_page(vma_of_vmf, vmf->address, last_pg); + // [!] YOU DON'T NEED TO CALL REMAP_PFN_RANGE OR FAMILY HERE!!! + // `__do_fault` allocates PTE prior to calling `vm_ops->fault`, + // and at return `finish_fault` inserts PTE for given page. + // [?] I think `vmf_insert_page` etc. are used to insert device pages... + // They require VM_MIXEDMAP, which in this case don't need (and in fact causes a BUG here.) + // I think this is also the reason why `remap_pfn_range` might work here, + // exactly because it does NOT try to add VM_MIXEDMAP, etc. _dbg_phys_of_page = page_to_phys(last_pg); mutex_unlock(&my_shmem_pages_mtx); @@ -164,7 +168,7 @@ err_ret_no_page: __func__); return VM_FAULT_OOM; ok_ret_allocated: - return 0; + return ret; } static const struct vm_operations_struct my_shmem_vmops = { diff --git a/src/aarch64-linux-flush-dcache/userspace/writer_null.c b/src/aarch64-linux-flush-dcache/userspace/writer_null.c index 7b8cf4b..a5bbb81 100644 --- a/src/aarch64-linux-flush-dcache/userspace/writer_null.c +++ b/src/aarch64-linux-flush-dcache/userspace/writer_null.c @@ -10,14 +10,34 @@ const char *DEVICE_PATH = "/dev/my_shmem"; +int parse_argument(int argc, char *argv[], size_t *parsed_arg) { + if (argc != 2) { + eprintf( + "Usage: %s \n", argv[0]); + return 1; + } + + *parsed_arg = strtol(argv[1], NULL, 10); + if (errno) + return 1; + return 0; +} + int main(int argc, char *argv[]) { + // Set write & alloc amount + size_t page_count; + if (parse_argument(argc, argv, &page_count)) + exit(EXIT_FAILURE); + const long PAGE_SIZE = sysconf(_SC_PAGESIZE); if (PAGE_SIZE == -1) { eprintf("Error retrieving page size: %d.\n", errno); exit(EXIT_FAILURE); } + const size_t WRITE_AMNT = PAGE_SIZE * page_count; + // open device file w/ RW perms FILE *fp = fopen(DEVICE_PATH, "r+"); if (!fp) { @@ -31,7 +51,7 @@ int main(int argc, char *argv[]) } // mmap device - void *buf = mmap(NULL, PAGE_SIZE * 2, PROT_READ | PROT_WRITE, + void *buf = mmap(NULL, WRITE_AMNT, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (!buf) { eprintf("Error mmap-ing: %d.\n", errno); @@ -39,11 +59,12 @@ int main(int argc, char *argv[]) } // Write to mmap-ed device - char to_write[] = {0xca, 0xfe, 0xbe, 0xef, 0x00}; - memcpy(buf, to_write, strlen(to_write)); + char to_write[4] = {0xca, 0xfe, 0xbe, 0xef}; + for (size_t byte_offset = 0; byte_offset < WRITE_AMNT; byte_offset += 4) + memcpy(buf, to_write, 4); // Unmap device - munmap(buf, PAGE_SIZE * 2); + munmap(buf, WRITE_AMNT); // Close device fclose(fp); diff --git a/src/rdma-test/.vscode-ctags b/src/rdma-test/.vscode-ctags new file mode 100644 index 0000000..8804dd9 --- /dev/null +++ b/src/rdma-test/.vscode-ctags @@ -0,0 +1,38 @@ +!_TAG_EXTRA_DESCRIPTION anonymous /Include tags for non-named objects like lambda/ +!_TAG_EXTRA_DESCRIPTION fileScope /Include tags of file scope/ +!_TAG_EXTRA_DESCRIPTION pseudo /Include pseudo tags/ +!_TAG_EXTRA_DESCRIPTION subparser /Include tags generated by subparsers/ +!_TAG_FIELD_DESCRIPTION epoch /the last modified time of the input file (only for F\/file kind tag)/ +!_TAG_FIELD_DESCRIPTION file /File-restricted scoping/ +!_TAG_FIELD_DESCRIPTION input /input file/ +!_TAG_FIELD_DESCRIPTION name /tag name/ +!_TAG_FIELD_DESCRIPTION pattern /pattern/ +!_TAG_FIELD_DESCRIPTION typeref /Type and name of a variable or typedef/ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_KIND_DESCRIPTION!C d,macro /macro definitions/ +!_TAG_KIND_DESCRIPTION!C e,enumerator /enumerators (values inside an enumeration)/ +!_TAG_KIND_DESCRIPTION!C f,function /function definitions/ +!_TAG_KIND_DESCRIPTION!C g,enum /enumeration names/ +!_TAG_KIND_DESCRIPTION!C h,header /included header files/ +!_TAG_KIND_DESCRIPTION!C m,member /struct, and union members/ +!_TAG_KIND_DESCRIPTION!C s,struct /structure names/ +!_TAG_KIND_DESCRIPTION!C t,typedef /typedefs/ +!_TAG_KIND_DESCRIPTION!C u,union /union names/ +!_TAG_KIND_DESCRIPTION!C v,variable /variable definitions/ +!_TAG_OUTPUT_EXCMD mixed /number, pattern, mixed, or combineV2/ +!_TAG_OUTPUT_FILESEP slash /slash or backslash/ +!_TAG_OUTPUT_MODE u-ctags /u-ctags or e-ctags/ +!_TAG_OUTPUT_VERSION 0.0 /current.age/ +!_TAG_PARSER_VERSION!C 1.1 /current.age/ +!_TAG_PATTERN_LENGTH_LIMIT 96 /0 for no limit/ +!_TAG_PROC_CWD /home/rubberhead/Git/00-UOE/unnamed_ba_thesis/src/rdma-test/ // +!_TAG_PROGRAM_AUTHOR Universal Ctags Team // +!_TAG_PROGRAM_NAME Universal Ctags /Derived from Exuberant Ctags/ +!_TAG_PROGRAM_URL https://ctags.io/ /official site/ +!_TAG_PROGRAM_VERSION 6.1.0 /v6.1.0/ +!_TAG_ROLE_DESCRIPTION!C!function foreigndecl /declared in foreign languages/ +!_TAG_ROLE_DESCRIPTION!C!header local /local header/ +!_TAG_ROLE_DESCRIPTION!C!header system /system header/ +!_TAG_ROLE_DESCRIPTION!C!macro undef /undefined/ +!_TAG_ROLE_DESCRIPTION!C!struct foreigndecl /declared in foreign languages/ diff --git a/src/rdma-test/main.c b/src/rdma-test/main.c new file mode 100644 index 0000000..10792b4 --- /dev/null +++ b/src/rdma-test/main.c @@ -0,0 +1,41 @@ +#include +#include +#include + +#include +#include +#include + +#define READ_ONCE(x) ({ \ + typeof(x) val; \ + __atomic_thread_fence(__ATOMIC_ACQUIRE); \ + val = (x); \ + __atomic_thread_fence(__ATOMIC_ACQUIRE); \ + val; \ +}) + +struct ib_res_struct { + struct ibv_context *ctx; + struct ibv_pd *pd; // Protection Domain + struct ibv_mr *mr; // Memory Registration + struct ibv_cq *cq; // Completion Queue + struct ibv_qp **qp; // Queue Pair (send, recv) + struct ibv_srq *srq; // Shared Receive Queue + struct ibv_port_attr port_attr; + struct ibv_device_attr dev_attr; + + int nr_qps; + char *ib_buf; + size_t ib_buf_size; +}; + +struct ib_res_struct IB_RES; + +int run_as_server() { + // byte buffer in ib_res + char *buf = IB_RES.ib_buf; + + char prev_char = *buf; + while(READ_ONCE(*buf) != 0x8f) { + } +} \ No newline at end of file