freebsd-nq/sys/contrib/octeon-sdk/cvmx-shmem.c

/***********************license start***************
 * Copyright (c) 2003-2010  Cavium Inc. (support@cavium.com). All rights
 * reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.

 *   * Neither the name of Cavium Inc. nor the names of
 *     its contributors may be used to endorse or promote products
 *     derived from this software without specific prior written
 *     permission.

 * This Software, including technical data, may be subject to U.S. export  control
 * laws, including the U.S. Export Administration Act and its  associated
 * regulations, and may be subject to export or import  regulations in other
 * countries.

 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
 ***********************license end**************************************/



/**
 * @file
 *   cvmx-shmem supplies the cross application shared memory implementation
 *
 * <hr>$Revision: 41586 $<hr>
 */
#include "cvmx.h"
#include "cvmx-bootmem.h"
#include "cvmx-tlb.h"
#include "cvmx-shmem.h"

//#define DEBUG

struct cvmx_shmem_smdr *__smdr = NULL;

#ifdef CVMX_BUILD_FOR_LINUX_USER
static int __cvmx_shmem_devmemfd = 0;   /* fd for /dev/mem */
#endif

#define __CHECK_APP_SMDR  do { \
                if (__smdr == NULL) { \
                    cvmx_dprintf("cvmx_shmem: %s is not set up, Quit line %d \n", \
                        CVMX_SHMEM_DSCPTR_NAME, __LINE__ ); \
                    exit(-1); \
                } \
              }while(0)



/**
 * @INTERNAL
 * Virtual sbrk, assigning virtual address in a global virtual address space.
 *
 * @param alignment   alignment requirement in bytes
 * @param size        size in bytes
 */
static inline void *__cvmx_shmem_vsbrk_64(uint64_t alignment, uint64_t size)
{
    uint64_t nbase_64 = CAST64(__smdr->break64);
    void *nbase = NULL;

    /* Skip unaligned bytes */
    if (nbase_64 & alignment)
        nbase_64 += ~(nbase_64 & alignment) + 1;

    if (nbase_64 + size  <  CVMX_SHMEM_VADDR64_END)
    {
        nbase = CASTPTR(void *, nbase_64);
        __smdr->break64 = nbase + size;
    }

    return nbase;
}

/**
 * @INTERNAL
 * Initialize all SMDR entries, only need to be called once
 *
 * @param smdr pointer to the SMDR
 */
static inline void __smdr_new(struct cvmx_shmem_smdr *smdr) {

    if (smdr != NULL)
    {
        int i;

        cvmx_spinlock_init (&smdr->lock);
        cvmx_spinlock_lock (&smdr->lock);

        for ( i = 0; i < CVMX_SHMEM_NUM_DSCPTR; i++ )
        {
            smdr -> shmd[i].owner = CVMX_SHMEM_OWNER_NONE;
            smdr -> shmd[i].is_named_block = 0;
            smdr -> shmd[i].use_count = 0;
            smdr -> shmd[i].name = NULL;
            smdr -> shmd[i].vaddr = NULL;
            smdr -> shmd[i].paddr = 0;
            smdr -> shmd[i].size = 0;
            smdr -> shmd[i].alignment = 0;
        };

        /* Init vaddr */
        smdr->break64 = (void *)CVMX_SHMEM_VADDR64_START;
        cvmx_spinlock_unlock (&smdr->lock);
    }

    /* Make sure the shmem descriptor region is created */
    __CHECK_APP_SMDR;
};



/**
 * @INTERNAL
 * Initialize __smdr pointer, if SMDR exits already. If not, create a new
 * one.  Once SMDR is created (as a bootmem named block), it is persistent.
 */
static inline struct cvmx_shmem_smdr *__smdr_init()
{
    const cvmx_bootmem_named_block_desc_t *smdr_nblk = NULL;
    size_t smdr_size = sizeof(*__smdr);
    char *smdr_name = CVMX_SHMEM_DSCPTR_NAME;

    __smdr = (struct cvmx_shmem_smdr *) cvmx_bootmem_alloc_named(smdr_size, 0x10000, smdr_name);

    if (__smdr)
       __smdr_new (__smdr);
    else
    {
        /* Check if SMDR exists already */
        smdr_nblk = cvmx_bootmem_find_named_block(smdr_name);
        if (smdr_nblk)
        {
            __smdr = (struct cvmx_shmem_smdr *)
            (cvmx_phys_to_ptr(smdr_nblk->base_addr));

            cvmx_spinlock_lock (&__smdr->lock);
            if (smdr_nblk->size != smdr_size)
            {
                cvmx_dprintf("SMDR named block is created by another "
                    "application with different size %lu, "
                    "expecting %lu \n",
                    (long unsigned int)smdr_nblk->size, (long unsigned int)smdr_size);
                __smdr = NULL;
            }
            cvmx_spinlock_unlock (&__smdr->lock);
        }
    }

   if (!__smdr)
       cvmx_dprintf("cvmx_shmem: Failed to allocate or find SMDR from bootmem \n");

   return __smdr;
};


/**
 * @INTERNAL
 * Generic Iterator function for all SMDR entries
 *
 * @param void(*f)(dscptr) the function to be invoked for every descriptor
 * @param param
 *
 * @return the descriptor iterator stopped at.
 */
static struct cvmx_shmem_dscptr *__smdr_iterator(struct cvmx_shmem_dscptr *(*f)(struct cvmx_shmem_dscptr *dscptr, void *p), void *param )
{
    struct cvmx_shmem_dscptr *d, *dscptr = NULL;
    int i;

    __CHECK_APP_SMDR;

    for (i = 0; i < CVMX_SHMEM_NUM_DSCPTR; i++)
    {
        d = &__smdr->shmd[i];
        if ((dscptr = (*f)(d, param)) != NULL)
            break;      /* stop iteration */
    }

   return dscptr;
}


/**
 * @INTERNAL
 * SMDR name match functor. to be used for iterator.
 *
 * @param dscptr  descriptor passed in by the iterator
 * @param   name    string to match against
 *
 * @return !NULL   descriptor matched
 *     NULL    not match
 */
static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_match_name(struct cvmx_shmem_dscptr *dscptr, void *name)
{
    char *name_to_match = (char *) name;
    struct cvmx_shmem_dscptr *ret = NULL;

    if (dscptr->owner == CVMX_SHMEM_OWNER_NONE)
        return NULL;

    if (strcmp(dscptr->name, name_to_match) == 0)
        ret =  dscptr;

    return ret;
}

/**
 * @INTERNAL
 * Find by name
 *
 * @param   name    string to match against
 *
 * @return !NULL    descriptor matched
 *          NULL    not match
 */
static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_find_by_name(char *name)
{
    return __smdr_iterator( __cvmx_shmem_smdr_match_name, name);
}

/**
 * @INTERNAL
 * SMDR is free functor. to be used for iterator.
 *
 * @param dscptr  descriptor passed in by the iterator
 * @param nouse
 *
 * @return !NULL  descriptor is free
 *          NULL  descriptor is not free
 */
static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_is_free(struct cvmx_shmem_dscptr* dscptr, void *nouse)
{
    if (dscptr->owner == CVMX_SHMEM_OWNER_NONE)
        return dscptr;
    else
        return NULL;
}

/**
 * @INTERNAL
 * Search SMDR to find the first free descriptor
 *
 * @return !NULL   free descriptor found
 *     NULL    nothing found
 */
struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_find_free_dscptr(void)
{
    return __smdr_iterator(__cvmx_shmem_smdr_is_free, NULL);
}

/**
 * @INTERNAL
 * free a descriptor
 *
 * @param dscptr  descriptor to be freed
 */
static void __cvmx_shmem_smdr_free(struct cvmx_shmem_dscptr *dscptr)
{
    dscptr->owner = CVMX_SHMEM_OWNER_NONE;
}


/**
 * Per core shmem init function
 *
 * @return  cvmx_shmem_smdr*   pointer to __smdr
 */
struct cvmx_shmem_smdr *cvmx_shmem_init()
{
    return __smdr_init();
}

/**
 * Open shared memory based on named block
 *
 * @return  dscptr descriptor of the opened named block
 */
struct cvmx_shmem_dscptr *cvmx_shmem_named_block_open(char *name, uint32_t size, int oflag)
{
    const cvmx_bootmem_named_block_desc_t *shmem_nblk = NULL;
    struct cvmx_shmem_dscptr *dscptr = NULL;
    int nblk_allocated = 0; /* Assume we don't need to allocate a new
                               bootmem block */
    void *vaddr = NULL;
    const uint64_t size_4k = 4*1024, size_512mb = 512*1024*1024;

    __CHECK_APP_SMDR;

    /* Check size, Make sure it is minimal 4K, no bigger than 512MB */
    if (size > size_512mb) {
        cvmx_dprintf("Shared memory size can not be bigger than 512MB \n");
        return NULL;
    }
    if (size < size_4k)
        size = size_4k;

    size = __upper_power_of_two(size);

    cvmx_spinlock_lock(&__smdr->lock);

    shmem_nblk = cvmx_bootmem_find_named_block(name);
    if ((shmem_nblk == NULL) &&  (oflag & CVMX_SHMEM_O_CREAT))
    {
       void *p;
       /* The named block does not exist, create it if caller specifies
          the O_CREAT flag */
        nblk_allocated = 1;
        p = cvmx_bootmem_alloc_named(size, size, name);
        if (p)
            shmem_nblk = cvmx_bootmem_find_named_block(name);
#ifdef DEBUG
        cvmx_dprintf("cvmx-shmem-dbg:"
             "creating a new block %s: blk %p, shmem_nblk %p \n",
             name, p, shmem_nblk);
#endif
    }

    if (shmem_nblk == NULL)
        goto err;

    /* We are now holding a valid named block */

    dscptr = __cvmx_shmem_smdr_find_by_name(name);
    if (dscptr)
    {
        if (nblk_allocated)
        {
            /* name conflict between bootmem name space and SMDR name space */
            cvmx_dprintf("cvmx-shmem: SMDR descriptor name conflict, %s \n", name);
            goto err;
        }
        /* Make sure size and alignment matches with existing descriptor */
        if ((size != dscptr->size) ||  (size != dscptr -> alignment))
            goto err;
    }
    else
    {
        /* Create a new descriptor */
        dscptr = __cvmx_shmem_smdr_find_free_dscptr();
        if (dscptr)
            goto init;
        else
        {
            cvmx_dprintf("cvmx-shmem: SMDR out of descriptors \n");
            goto err;
        }
    }

    /* Maintain the reference count */
    if (dscptr != NULL)
        dscptr->use_count += 1;

    cvmx_spinlock_unlock(&__smdr->lock);
    return dscptr;

err:
#ifdef DEBUG
    cvmx_dprintf("cvmx-shmem-dbg: named block open failed \n");
#endif

    if (dscptr)
        __cvmx_shmem_smdr_free(dscptr);
    if (shmem_nblk && nblk_allocated)
        cvmx_bootmem_free_named(name);
    cvmx_spinlock_unlock(&__smdr->lock);

    return NULL;

init:

#ifdef DEBUG
    cvmx_dprintf("cvmx-shmem-dbg: init SMDR descriptor %p \n", dscptr);
#endif

    /* Assign vaddr for single address space mapping */
    vaddr = __cvmx_shmem_vsbrk_64(size, size);
    if (vaddr == NULL) {
        /* Failed to allocate virtual address, clean up */
        goto err;
    }

#ifdef DEBUG
    cvmx_dprintf("cmvx-shmem-dbg: allocated vaddr %p \n", vaddr);
#endif
    dscptr->vaddr = vaddr;

    /* Store descriptor information,  name, alignment,size... */
    dscptr->owner = cvmx_get_core_num();
    dscptr->is_named_block = 1;
    dscptr->use_count = 1;
    dscptr->name =shmem_nblk->name ;
    dscptr->paddr = shmem_nblk->base_addr;
    dscptr->size = size;
    dscptr->alignment = size;

    /* Store permission bits */
    if (oflag & CVMX_SHMEM_O_WRONLY)
        dscptr->p_wronly = 1;
    if (oflag & CVMX_SHMEM_O_RDWR)
        dscptr->p_rdwr = 1;

   cvmx_spinlock_unlock(&__smdr->lock);
   return dscptr;
}

/**
 * @INTERNAL
 *
 *  For stand along SE application only.
 *
 *  Add TLB mapping to map the shared memory
 *
 *  @param dscptr  shared memory descriptor
 *  @param pflag   protection flags
 *
 *  @return vaddr  the virtual address mapped for the shared memory
 */
#ifndef CVMX_BUILD_FOR_LINUX_USER
void *__cvmx_shmem_map_standalone(struct cvmx_shmem_dscptr *dscptr, int pflag)
{
    int free_index;

    /* Find a free tlb entry */
    free_index = cvmx_tlb_allocate_runtime_entry();

    if (free_index < 0 )
    {
        cvmx_dprintf("cvmx-shmem: shmem_map failed, out TLB entries \n");
        return NULL;
    }

#ifdef DEBUG
    cvmx_dprintf("cmvx-shmem-dbg:"
         "shmem_map TLB %d: vaddr %p paddr %lx, size %x \n",
         free_index, dscptr->vaddr, dscptr->paddr, dscptr->size );
#endif

    cvmx_tlb_write_runtime_entry(free_index, CAST64(dscptr->vaddr),
            dscptr->paddr, dscptr->size,
            TLB_DIRTY | TLB_VALID | TLB_GLOBAL);

    return dscptr -> vaddr;
}
#endif

/**
 * @INTERNAL
 *
 *  For Linux user application only
 *
 *  Add mmap the shared memory
 *
 *  @param dscptr  shared memory descriptor
 *  @param pflag   protection flags
 *
 *  @return vaddr  the virtual address mapped for the shared memory
 */
#ifdef CVMX_BUILD_FOR_LINUX_USER
static inline void *__cvmx_shmem_map_linux(struct cvmx_shmem_dscptr *dscptr, int pflag)
{
    void *vaddr = NULL;

    if(__cvmx_shmem_devmemfd == 0)
    {
        __cvmx_shmem_devmemfd = open("/dev/mem", O_RDWR);
        if (__cvmx_shmem_devmemfd < 0)
        {
            cvmx_dprintf("Failed to open /dev/mem\n");
            exit(-1);
        }
    }

    vaddr = mmap(dscptr->vaddr, dscptr->size, PROT_READ|PROT_WRITE,
                 MAP_SHARED, __cvmx_shmem_devmemfd, 0);

    /* Make sure the mmap maps to the same virtual address specified in
     * descriptor
     */
    if ((vaddr!=NULL) && (vaddr != dscptr->vaddr))
    {
        munmap(vaddr, dscptr->size);
        vaddr = NULL;
    }
    return vaddr;
}
#endif

/**
 *  cvmx_shmem API
 *
 *  Add mapping for the shared memory
 *
 *  @param dscptr  shared memory descriptor
 *  @param pflag   protection flags
 *
 *  @return vaddr  the virtual address mapped for the shared memory
 */
void *cvmx_shmem_map(struct cvmx_shmem_dscptr *dscptr, int pflag)
{
    void *vaddr = NULL;
#ifdef CVMX_BUILD_FOR_LINUX_USER
    vaddr = __cvmx_shmem_map_linux(dscptr, pflag);
#else
    vaddr = __cvmx_shmem_map_standalone(dscptr, pflag);
#endif
    return vaddr;
}


/**
 * @INTERNAL
 *
 *  For Linux user application only
 *
 *  ummap the shared memory
 *
 *  @param dscptr  shared memory descriptor
 *
 */
#ifdef CVMX_BUILD_FOR_LINUX_USER
static inline void __cvmx_shmem_unmap_linux(struct cvmx_shmem_dscptr* dscptr)
{
    if (__cvmx_shmem_devmemfd && dscptr)
        munmap(dscptr->vaddr, dscptr->size);
}
#endif


/**
 * @INTERNAL
 *
 *  For stand along SE application only.
 *
 *  ummap the shared memory
 *
 *  @param dscptr  shared memory descriptor
 *
 */
#ifndef CVMX_BUILD_FOR_LINUX_USER
static inline void
__cvmx_shmem_unmap_standalone(struct cvmx_shmem_dscptr *dscptr)
{
    int index;

    index = cvmx_tlb_lookup(CAST64(dscptr->vaddr));

#ifdef DEBUG
    cvmx_dprintf("cmvx-shmem-dbg:"
             "shmem_unmap TLB %d \n", index);
#endif
    cvmx_tlb_free_runtime_entry(index);
}
#endif

/**
 *  ummap the shared memory
 *
 *  @param dscptr  shared memory descriptor
 *
 */
void cvmx_shmem_unmap(struct cvmx_shmem_dscptr *dscptr)
{
#ifdef CVMX_BUILD_FOR_LINUX_USER
    __cvmx_shmem_unmap_linux(dscptr);
#else
    __cvmx_shmem_unmap_standalone(dscptr);
#endif
}

/**
 * @INTERNAL
 *
 *  Common implementation of closing a descriptor.
 *
 *  @param dscptr  shared memory descriptor
 *  @param remove  1:  remove the descriptor and named block if this
 *                  this is the last user of the descriptor
 *             0:  do not remove
 *  @return  0:   Success
 *          !0:   Failed
 *
 */
static inline int __cvmx_shmem_close_dscptr(struct cvmx_shmem_dscptr *dscptr, int remove)
{
    cvmx_spinlock_lock(&dscptr->lock);

    if (dscptr->use_count >0)
        dscptr->use_count-= 1;

    if ((dscptr->use_count == 0) && remove)
    {
        /* Free this descriptor */
        __cvmx_shmem_smdr_free(dscptr);

        /* Free named block if this is the last user, and the block
           is created by the application */
        if (dscptr->is_named_block)
        {
#ifdef DEBUG
            cvmx_dprintf("cvmx-shmem-dbg: remove named block %s \n", dscptr->name);
#endif
            cvmx_bootmem_phy_named_block_free(dscptr->name, 0);
        }
    }
    cvmx_spinlock_unlock(&dscptr->lock);
    return 0;
}


/**
 * @INTERNAL
 *
 *  For stand along SE application only.
 *
 *  close a descriptor.
 *
 *  @param dscptr  shared memory descriptor
 *  @param remove  1:  remove the descriptor and named block if this
 *                  this is the last user of the descriptor
 *             0:  do not remove
 *  @return  0:   Success
 *          !0:   Failed
 *
 */
#ifndef CVMX_BUILD_FOR_LINUX_USER
static inline int __cvmx_shmem_close_standalone(struct cvmx_shmem_dscptr *dscptr, int remove)
{
    return __cvmx_shmem_close_dscptr(dscptr, remove);
}
#endif

/**
 * @INTERNAL
 *
 *  For Linux user application only.
 *
 *  close a descriptor.
 *
 *  @param dscptr  shared memory descriptor
 *  @param remove  1:  remove the descriptor and named block if this
 *                  this is the last user of the descriptor
 *             0:  do not remove
 *  @return  0:   Success
 *          !0:   Failed
 *
 */
#ifdef CVMX_BUILD_FOR_LINUX_USER
int __cvmx_shmem_close_linux(struct cvmx_shmem_dscptr *dscptr, int remove)
{
    int ret;
    ret = __cvmx_shmem_close_dscptr(dscptr, remove);

    if (ret && __cvmx_shmem_devmemfd)
    {
        close(__cvmx_shmem_devmemfd);
         __cvmx_shmem_devmemfd=0;
    }

    return ret;

}
#endif

/**
 *
 *  close a descriptor.
 *
 *  @param dscptr  shared memory descriptor
 *  @param remove  1:  remove the descriptor and named block if this
 *                  this is the last user of the descriptor
 *             0:  do not remove
 *  @return  0:   Success
 *          !0:   Failed
 *
 */
int cvmx_shmem_close(struct cvmx_shmem_dscptr *dscptr, int remove)
{
    int ret;
#ifdef CVMX_BUILD_FOR_LINUX_USER
    ret = __cvmx_shmem_close_linux(dscptr, remove);
#else
    ret = __cvmx_shmem_close_standalone(dscptr, remove);
#endif
    return ret;
}

#ifdef DEBUG
/**
 * @INTERNAL
 *  SMDR non-free descriptor dump functor. to be used for iterator.
 *
 * @param dscptr  descriptor passed in by the iterator
 *
 * @return NULL  always
 */
static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_display_dscptr(struct cvmx_shmem_dscptr *dscptr, void *nouse)
{
    if ((dscptr != NULL ) && (dscptr -> owner != CVMX_SHMEM_OWNER_NONE))
    {
        cvmx_dprintf("  %s: phy: %lx, size %d, alignment %lx, virt %p use_count %d\n",
            dscptr->name, dscptr-> paddr,
            dscptr->size, dscptr-> alignment,
            dscptr->vaddr, dscptr->use_count);
    }

    return NULL;
}
#endif

/**
 *  SMDR descriptor show
 *
 *  list all non-free descriptors
 */
void cvmx_shmem_show(void)
{
    __CHECK_APP_SMDR;

#ifdef DEBUG
    cvmx_dprintf("SMDR descriptor list: \n");
    cvmx_spinlock_lock(&__smdr->lock);
    __smdr_iterator(__cvmx_shmem_smdr_display_dscptr, NULL);
    cvmx_spinlock_unlock(&__smdr->lock);
    cvmx_dprintf("\n\n");
#endif
}