/* Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and * only version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * RMNET Data Smart Hash Workqueue solution * */ #include "rmnet_shs.h" #include "rmnet_shs_wq_genl.h" #include "rmnet_shs_wq_mem.h" #include #include #include #include MODULE_LICENSE("GPL v2"); /* Local Macros */ #define RMNET_SHS_RX_BPNSEC_TO_BPSEC(x) ((x)*1000000000) #define RMNET_SHS_SEC_TO_NSEC(x) ((x)*1000000000) #define RMNET_SHS_NSEC_TO_SEC(x) ((x)/1000000000) #define RMNET_SHS_BYTE_TO_BIT(x) ((x)*8) #define RMNET_SHS_MIN_HSTAT_NODES_REQD 16 #define RMNET_SHS_FILTER_PKT_LIMIT 200 #define RMNET_SHS_FILTER_FLOW_RATE 100 #define PERIODIC_CLEAN 0 /* FORCE_CLEAN should only used during module de-init.*/ #define FORCE_CLEAN 1 /* Local Definitions and Declarations */ unsigned int rmnet_shs_cpu_prio_dur __read_mostly = 3; module_param(rmnet_shs_cpu_prio_dur, uint, 0644); MODULE_PARM_DESC(rmnet_shs_cpu_prio_dur, "Priority ignore duration (wq intervals)"); #define PRIO_BACKOFF ((!rmnet_shs_cpu_prio_dur) ? 2 : rmnet_shs_cpu_prio_dur) #define RMNET_SHS_SEGS_PER_SKB_DEFAULT (2) unsigned int rmnet_shs_wq_interval_ms __read_mostly = RMNET_SHS_WQ_INTERVAL_MS; module_param(rmnet_shs_wq_interval_ms, uint, 0644); MODULE_PARM_DESC(rmnet_shs_wq_interval_ms, "Interval between wq runs (ms)"); unsigned long rmnet_shs_max_flow_inactivity_sec __read_mostly = RMNET_SHS_MAX_SKB_INACTIVE_TSEC; module_param(rmnet_shs_max_flow_inactivity_sec, ulong, 0644); MODULE_PARM_DESC(rmnet_shs_max_flow_inactivity_sec, "Max flow inactive time before clean up"); unsigned int rmnet_shs_wq_tuning __read_mostly = 80; module_param(rmnet_shs_wq_tuning, uint, 0644); MODULE_PARM_DESC(rmnet_shs_wq_tuning, "moving average weightage"); unsigned long long rmnet_shs_cpu_rx_max_pps_thresh[MAX_CPUS]__read_mostly = { RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH, RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH, RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH, RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH }; module_param_array(rmnet_shs_cpu_rx_max_pps_thresh, ullong, 0, 0644); MODULE_PARM_DESC(rmnet_shs_cpu_rx_max_pps_thresh, "Max pkts core can handle"); unsigned long long rmnet_shs_cpu_rx_min_pps_thresh[MAX_CPUS]__read_mostly = { RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH, RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH, RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH, RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH, RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH }; module_param_array(rmnet_shs_cpu_rx_min_pps_thresh, ullong, 0, 0644); MODULE_PARM_DESC(rmnet_shs_cpu_rx_min_pps_thresh, "Min pkts core can handle"); unsigned int rmnet_shs_cpu_rx_flows[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_flows, uint, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_flows, "Num flows processed per core"); unsigned int rmnet_shs_cpu_rx_filter_flows[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_filter_flows, uint, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_filter_flows, "Num filtered flows per core"); unsigned long long rmnet_shs_cpu_rx_bytes[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_bytes, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_bytes, "SHS stamp bytes per CPU"); unsigned long long rmnet_shs_cpu_rx_pkts[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_pkts, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_pkts, "SHS stamp total pkts per CPU"); unsigned long long rmnet_shs_cpu_rx_bps[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_bps, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_bps, "SHS stamp enq rate per CPU"); unsigned long long rmnet_shs_cpu_rx_pps[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_pps, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_pps, "SHS stamp pkt enq rate per CPU"); unsigned long long rmnet_shs_cpu_qhead_diff[MAX_CPUS]; module_param_array(rmnet_shs_cpu_qhead_diff, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_qhead_diff, "SHS nw stack queue processed diff"); unsigned long long rmnet_shs_cpu_qhead_total[MAX_CPUS]; module_param_array(rmnet_shs_cpu_qhead_total, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_qhead_total, "SHS nw stack queue processed total"); unsigned long rmnet_shs_flow_hash[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_hash, ulong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_hash, "SHS stamp hash flow"); unsigned long rmnet_shs_flow_proto[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_proto, ulong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_proto, "SHS stamp hash transport protocol"); unsigned long long rmnet_shs_flow_inactive_tsec[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_inactive_tsec, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_inactive_tsec, "SHS stamp inactive flow time"); int rmnet_shs_flow_cpu[MAX_SUPPORTED_FLOWS_DEBUG] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; module_param_array(rmnet_shs_flow_cpu, int, NULL, 0444); MODULE_PARM_DESC(rmnet_shs_flow_cpu, "SHS stamp flow processing CPU"); int rmnet_shs_flow_cpu_recommended[MAX_SUPPORTED_FLOWS_DEBUG] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; module_param_array(rmnet_shs_flow_cpu_recommended, int, NULL, 0444); MODULE_PARM_DESC(rmnet_shs_flow_cpu_recommended, "SHS stamp flow proc CPU"); unsigned long long rmnet_shs_flow_rx_bytes[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_rx_bytes, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_rx_bytes, "SHS stamp bytes per flow"); unsigned long long rmnet_shs_flow_rx_pkts[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_rx_pkts, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_rx_pkts, "SHS stamp total pkts per flow"); unsigned long long rmnet_shs_flow_rx_bps[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_rx_bps, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_rx_bps, "SHS stamp enq rate per flow"); unsigned long long rmnet_shs_flow_rx_pps[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_rx_pps, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_rx_pps, "SHS stamp pkt enq rate per flow"); /* Counters for suggestions made by wq */ unsigned long long rmnet_shs_flow_silver_to_gold[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_silver_to_gold, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_silver_to_gold, "SHS Suggest Silver to Gold"); unsigned long long rmnet_shs_flow_gold_to_silver[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_gold_to_silver, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_gold_to_silver, "SHS Suggest Gold to Silver"); unsigned long long rmnet_shs_flow_gold_balance[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_gold_balance, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_gold_balance, "SHS Suggest Gold Balance"); static DEFINE_SPINLOCK(rmnet_shs_hstat_tbl_lock); static DEFINE_SPINLOCK(rmnet_shs_ep_lock); static time_t rmnet_shs_wq_tnsec; static struct workqueue_struct *rmnet_shs_wq; static struct rmnet_shs_delay_wq_s *rmnet_shs_delayed_wq; static struct rmnet_shs_wq_rx_flow_s rmnet_shs_rx_flow_tbl; static struct list_head rmnet_shs_wq_hstat_tbl = LIST_HEAD_INIT(rmnet_shs_wq_hstat_tbl); static int rmnet_shs_flow_dbg_stats_idx_cnt; struct list_head rmnet_shs_wq_ep_tbl = LIST_HEAD_INIT(rmnet_shs_wq_ep_tbl); /* Helper functions to add and remove entries to the table * that maintains a list of all endpoints (vnd's) available on this device. */ void rmnet_shs_wq_ep_tbl_add(struct rmnet_shs_wq_ep_s *ep) { trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL); list_add(&ep->ep_list_id, &rmnet_shs_wq_ep_tbl); } void rmnet_shs_wq_ep_tbl_remove(struct rmnet_shs_wq_ep_s *ep) { trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_DEL, 0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL); list_del_init(&ep->ep_list_id); } /* Helper functions to add and remove entries to the table * that maintains a list of all nodes that maintain statistics per flow */ void rmnet_shs_wq_hstat_tbl_add(struct rmnet_shs_wq_hstat_s *hnode) { unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_add(&hnode->hstat_node_id, &rmnet_shs_wq_hstat_tbl); spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_hstat_tbl_remove(struct rmnet_shs_wq_hstat_s *hnode) { unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_DEL, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_del_init(&hnode->hstat_node_id); spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } /* We maintain a list of all flow nodes processed by a cpu. * Below helper functions are used to maintain flow<=>cpu * association.* */ void rmnet_shs_wq_cpu_list_remove(struct rmnet_shs_wq_hstat_s *hnode) { unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_DEL, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_del_init(&hnode->cpu_node_id); spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_cpu_list_add(struct rmnet_shs_wq_hstat_s *hnode, struct list_head *head) { unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_add(&hnode->cpu_node_id, head); spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_cpu_list_move(struct rmnet_shs_wq_hstat_s *hnode, struct list_head *head) { unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_MOVE, hnode->current_cpu, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_move(&hnode->cpu_node_id, head); spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } /* Resets all the parameters used to maintain hash statistics */ void rmnet_shs_wq_hstat_reset_node(struct rmnet_shs_wq_hstat_s *hnode) { hnode->c_epoch = 0; hnode->l_epoch = 0; hnode->node = NULL; hnode->inactive_duration = 0; hnode->rx_skb = 0; hnode->rx_bytes = 0; hnode->rx_pps = 0; hnode->rx_bps = 0; hnode->last_rx_skb = 0; hnode->last_rx_bytes = 0; hnode->rps_config_msk = 0; hnode->current_core_msk = 0; hnode->def_core_msk = 0; hnode->pri_core_msk = 0; hnode->available_core_msk = 0; hnode->hash = 0; hnode->suggested_cpu = 0; hnode->current_cpu = 0; hnode->segs_per_skb = 0; hnode->skb_tport_proto = 0; hnode->stat_idx = -1; INIT_LIST_HEAD(&hnode->cpu_node_id); hnode->is_new_flow = 0; /* clear in use flag as a last action. This is required to ensure * the same node does not get allocated until all the paramaeters * are cleared. */ hnode->in_use = 0; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_NODE_RESET, hnode->is_perm, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); } /* Preallocates a set of flow nodes that maintain flow level statistics*/ void rmnet_shs_wq_hstat_alloc_nodes(u8 num_nodes_to_allocate, u8 is_store_perm) { struct rmnet_shs_wq_hstat_s *hnode = NULL; while (num_nodes_to_allocate > 0) { hnode = kzalloc(sizeof(*hnode), GFP_ATOMIC); if (hnode) { hnode->is_perm = is_store_perm; rmnet_shs_wq_hstat_reset_node(hnode); INIT_LIST_HEAD(&hnode->hstat_node_id); INIT_LIST_HEAD(&hnode->cpu_node_id); rmnet_shs_wq_hstat_tbl_add(hnode); } else { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_HSTAT_ERR]++; } hnode = NULL; num_nodes_to_allocate--; } } /* If there is an already pre-allocated node available and not in use, * we will try to re-use them. */ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void) { struct rmnet_shs_wq_hstat_s *hnode = NULL; struct rmnet_shs_wq_hstat_s *ret_node = NULL; unsigned long flags; spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (hnode == NULL) continue; if (hnode->in_use == 0) { ret_node = hnode; ret_node->in_use = 1; ret_node->is_new_flow = 1; break; } } spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); if (ret_node) { trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_NODE_REUSE, hnode->is_perm, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); return ret_node; } /* We have reached a point where all pre-allocated nodes are in use * Allocating memory to maintain the flow level stats for new flow. * However, this newly allocated memory will be released as soon as we * realize that this flow is inactive */ ret_node = kzalloc(sizeof(*hnode), GFP_ATOMIC); if (!ret_node) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_HSTAT_ERR]++; return NULL; } rmnet_shs_wq_hstat_reset_node(ret_node); ret_node->is_perm = 0; ret_node->in_use = 1; ret_node->is_new_flow = 1; INIT_LIST_HEAD(&ret_node->hstat_node_id); INIT_LIST_HEAD(&ret_node->cpu_node_id); trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_NODE_DYN_ALLOCATE, ret_node->is_perm, 0xDEF, 0xDEF, 0xDEF, ret_node, NULL); rmnet_shs_wq_hstat_tbl_add(ret_node); return ret_node; } void rmnet_shs_wq_create_new_flow(struct rmnet_shs_skbn_s *node_p) { struct timespec time; if (!node_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } node_p->hstats = rmnet_shs_wq_get_new_hstat_node(); if (node_p->hstats != NULL) { (void)getnstimeofday(&time); node_p->hstats->hash = node_p->hash; node_p->hstats->skb_tport_proto = node_p->skb_tport_proto; node_p->hstats->current_cpu = node_p->map_cpu; node_p->hstats->suggested_cpu = node_p->map_cpu; /* Set egmentation off by default */ node_p->hstats->segs_per_skb = 0; /* Start TCP flows with segmentation if userspace connected */ if (rmnet_shs_userspace_connected && node_p->hstats->skb_tport_proto == IPPROTO_TCP) node_p->hstats->segs_per_skb = RMNET_SHS_SEGS_PER_SKB_DEFAULT; node_p->hstats->node = node_p; node_p->hstats->c_epoch = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) + time.tv_nsec; node_p->hstats->l_epoch = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) + time.tv_nsec; } trace_rmnet_shs_wq_high(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_NODE_NEW_REQ, 0xDEF, 0xDEF, 0xDEF, 0xDEF, node_p, node_p->hstats); } /* Compute the average pps for a flow based on tuning param * Often when we decide to switch from a small cluster core, * it is because of the heavy traffic on that core. In such * circumstances, we want to switch to a big cluster * core as soon as possible. Therefore, we will provide a * greater weightage to the most recent sample compared to * the previous samples. * * On the other hand, when a flow which is on a big cluster * cpu suddenly starts to receive low traffic we move to a * small cluster core after observing low traffic for some * more samples. This approach avoids switching back and forth * to small cluster cpus due to momentary decrease in data * traffic. */ static u64 rmnet_shs_wq_get_flow_avg_pps(struct rmnet_shs_wq_hstat_s *hnode) { u64 avg_pps, mov_avg_pps; u16 new_weight, old_weight; if (!hnode) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return 0; } if (rmnet_shs_is_lpwr_cpu(hnode->current_cpu)) { /* More weight to current value */ new_weight = rmnet_shs_wq_tuning; old_weight = 100 - rmnet_shs_wq_tuning; } else { old_weight = rmnet_shs_wq_tuning; new_weight = 100 - rmnet_shs_wq_tuning; } /* computing weighted average per flow, if the flow has just started, * there is no past values, so we use the current pps as the avg */ if (hnode->last_pps == 0) { avg_pps = hnode->rx_pps; } else { mov_avg_pps = (hnode->last_pps + hnode->avg_pps) / 2; avg_pps = (((new_weight * hnode->rx_pps) + (old_weight * mov_avg_pps)) / (new_weight + old_weight)); } return avg_pps; } static u64 rmnet_shs_wq_get_cpu_avg_pps(u16 cpu_num) { u64 avg_pps, mov_avg_pps; u16 new_weight, old_weight; struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_node; struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl; if (cpu_num >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return 0; } cpu_node = &rx_flow_tbl_p->cpu_list[cpu_num]; if (rmnet_shs_is_lpwr_cpu(cpu_num)) { /* More weight to current value */ new_weight = rmnet_shs_wq_tuning; old_weight = 100 - rmnet_shs_wq_tuning; } else { old_weight = rmnet_shs_wq_tuning; new_weight = 100 - rmnet_shs_wq_tuning; } /* computing weighted average per flow, if the cpu has not past values * for pps, we use the current value as the average */ if (cpu_node->last_rx_pps == 0) { avg_pps = cpu_node->avg_pps; } else { mov_avg_pps = (cpu_node->last_rx_pps + cpu_node->avg_pps) / 2; avg_pps = (((new_weight * cpu_node->rx_pps) + (old_weight * mov_avg_pps)) / (new_weight + old_weight)); } trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_EVAL_CPU, cpu_num, cpu_node->rx_pps, cpu_node->last_rx_pps, avg_pps, NULL, NULL); return avg_pps; } /* Refresh the RPS mask associated with this flow */ void rmnet_shs_wq_update_hstat_rps_msk(struct rmnet_shs_wq_hstat_s *hstat_p) { struct rmnet_shs_skbn_s *node_p = NULL; struct rmnet_shs_wq_ep_s *ep = NULL; if (!hstat_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } node_p = hstat_p->node; /*Map RPS mask from the endpoint associated with this flow*/ list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (ep && (node_p->dev == ep->ep)) { hstat_p->rps_config_msk = ep->rps_config_msk; hstat_p->def_core_msk = ep->default_core_msk; hstat_p->pri_core_msk = ep->pri_core_msk; /* Update ep tput stats while we're here */ if (hstat_p->skb_tport_proto == IPPROTO_TCP) { rm_err("SHS_UDP: adding TCP bps %lu to ep_total %lu ep name %s", hstat_p->rx_bps, ep->tcp_rx_bps, node_p->dev->name); ep->tcp_rx_bps += hstat_p->rx_bps; } else if (hstat_p->skb_tport_proto == IPPROTO_UDP) { rm_err("SHS_UDP: adding UDP rx_bps %lu to ep_total %lu ep name %s", hstat_p->rx_bps, ep->udp_rx_bps, node_p->dev->name); ep->udp_rx_bps += hstat_p->rx_bps; } break; } } trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_UPDATE_MSK, hstat_p->rps_config_msk, hstat_p->def_core_msk, hstat_p->pri_core_msk, 0xDEF, hstat_p, node_p); } void rmnet_shs_wq_update_hash_stats_debug(struct rmnet_shs_wq_hstat_s *hstats_p, struct rmnet_shs_skbn_s *node_p) { int idx = rmnet_shs_flow_dbg_stats_idx_cnt; if (!rmnet_shs_stats_enabled) return; if (!hstats_p || !node_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } if (hstats_p->stat_idx < 0) { idx = idx % MAX_SUPPORTED_FLOWS_DEBUG; hstats_p->stat_idx = idx; rmnet_shs_flow_dbg_stats_idx_cnt++; } rmnet_shs_flow_hash[hstats_p->stat_idx] = hstats_p->hash; rmnet_shs_flow_proto[hstats_p->stat_idx] = node_p->skb_tport_proto; rmnet_shs_flow_inactive_tsec[hstats_p->stat_idx] = RMNET_SHS_NSEC_TO_SEC(hstats_p->inactive_duration); rmnet_shs_flow_rx_bps[hstats_p->stat_idx] = hstats_p->rx_bps; rmnet_shs_flow_rx_pps[hstats_p->stat_idx] = hstats_p->rx_pps; rmnet_shs_flow_rx_bytes[hstats_p->stat_idx] = hstats_p->rx_bytes; rmnet_shs_flow_rx_pkts[hstats_p->stat_idx] = hstats_p->rx_skb; rmnet_shs_flow_cpu[hstats_p->stat_idx] = hstats_p->current_cpu; rmnet_shs_flow_cpu_recommended[hstats_p->stat_idx] = hstats_p->suggested_cpu; rmnet_shs_flow_silver_to_gold[hstats_p->stat_idx] = hstats_p->rmnet_shs_wq_suggs[RMNET_SHS_WQ_SUGG_SILVER_TO_GOLD]; rmnet_shs_flow_gold_to_silver[hstats_p->stat_idx] = hstats_p->rmnet_shs_wq_suggs[RMNET_SHS_WQ_SUGG_GOLD_TO_SILVER]; rmnet_shs_flow_gold_balance[hstats_p->stat_idx] = hstats_p->rmnet_shs_wq_suggs[RMNET_SHS_WQ_SUGG_GOLD_BALANCE]; } /* Returns TRUE if this flow received a new packet * FALSE otherwise */ u8 rmnet_shs_wq_is_hash_rx_new_pkt(struct rmnet_shs_wq_hstat_s *hstats_p, struct rmnet_shs_skbn_s *node_p) { if (!hstats_p || !node_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return 0; } if (node_p->num_skb == hstats_p->rx_skb) return 0; return 1; } void rmnet_shs_wq_update_hash_tinactive(struct rmnet_shs_wq_hstat_s *hstats_p, struct rmnet_shs_skbn_s *node_p) { time_t tdiff; if (!hstats_p || !node_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } tdiff = rmnet_shs_wq_tnsec - hstats_p->c_epoch; hstats_p->inactive_duration = tdiff; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_FLOW_INACTIVE, hstats_p->hash, tdiff, 0xDEF, 0xDEF, hstats_p, NULL); } void rmnet_shs_wq_update_hash_stats(struct rmnet_shs_wq_hstat_s *hstats_p) { time_t tdiff; u64 skb_diff, bytes_diff; struct rmnet_shs_skbn_s *node_p; if (!hstats_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } node_p = hstats_p->node; if (!rmnet_shs_wq_is_hash_rx_new_pkt(hstats_p, node_p)) { hstats_p->rx_pps = 0; hstats_p->avg_pps = 0; hstats_p->rx_bps = 0; rmnet_shs_wq_update_hash_tinactive(hstats_p, node_p); rmnet_shs_wq_update_hash_stats_debug(hstats_p, node_p); return; } trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_START, hstats_p->hash, 0xDEF, hstats_p->rx_pps, hstats_p->rx_bps, hstats_p, NULL); rmnet_shs_wq_update_hstat_rps_msk(hstats_p); hstats_p->inactive_duration = 0; hstats_p->l_epoch = node_p->hstats->c_epoch; hstats_p->last_rx_skb = node_p->hstats->rx_skb; hstats_p->last_rx_bytes = node_p->hstats->rx_bytes; hstats_p->c_epoch = rmnet_shs_wq_tnsec; hstats_p->rx_skb = node_p->num_skb; hstats_p->rx_bytes = node_p->num_skb_bytes; tdiff = (hstats_p->c_epoch - hstats_p->l_epoch); skb_diff = hstats_p->rx_skb - hstats_p->last_rx_skb; bytes_diff = hstats_p->rx_bytes - hstats_p->last_rx_bytes; hstats_p->rx_pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(skb_diff)/(tdiff); hstats_p->rx_bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(bytes_diff)/(tdiff); hstats_p->rx_bps = RMNET_SHS_BYTE_TO_BIT(hstats_p->rx_bps); hstats_p->avg_pps = rmnet_shs_wq_get_flow_avg_pps(hstats_p); hstats_p->last_pps = hstats_p->rx_pps; rmnet_shs_wq_update_hash_stats_debug(hstats_p, node_p); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_END, hstats_p->hash, hstats_p->rx_pps, hstats_p->rx_bps, (tdiff/1000000), hstats_p, NULL); } static void rmnet_shs_wq_refresh_cpu_rates_debug(u16 cpu, struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_p) { if (!rmnet_shs_stats_enabled) return; if (cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return; } if (!cpu_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } rmnet_shs_cpu_rx_bps[cpu] = cpu_p->rx_bps; rmnet_shs_cpu_rx_pps[cpu] = cpu_p->rx_pps; rmnet_shs_cpu_rx_flows[cpu] = cpu_p->flows; rmnet_shs_cpu_rx_bytes[cpu] = cpu_p->rx_bytes; rmnet_shs_cpu_rx_pkts[cpu] = cpu_p->rx_skbs; rmnet_shs_cpu_qhead_diff[cpu] = cpu_p->qhead_diff; rmnet_shs_cpu_qhead_total[cpu] = cpu_p->qhead_total; } static void rmnet_shs_wq_refresh_dl_mrkr_stats(void) { struct rmnet_shs_wq_rx_flow_s *tbl_p = &rmnet_shs_rx_flow_tbl; struct rmnet_port *port; u64 pkt_diff, byte_diff; time_t tdiff; tbl_p->dl_mrk_last_rx_bytes = tbl_p->dl_mrk_rx_bytes; tbl_p->dl_mrk_last_rx_pkts = tbl_p->dl_mrk_rx_pkts; port = rmnet_shs_cfg.port; if (!port) { rmnet_shs_crit_err[RMNET_SHS_WQ_GET_RMNET_PORT_ERR]++; return; } tbl_p->dl_mrk_rx_pkts = port->stats.dl_hdr_total_pkts; tbl_p->dl_mrk_rx_bytes = port->stats.dl_hdr_total_bytes; tdiff = rmnet_shs_wq_tnsec - tbl_p->l_epoch; pkt_diff = tbl_p->dl_mrk_rx_pkts - tbl_p->dl_mrk_last_rx_pkts; byte_diff = tbl_p->dl_mrk_rx_bytes - tbl_p->dl_mrk_last_rx_bytes; tbl_p->dl_mrk_rx_pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(pkt_diff)/tdiff; tbl_p->dl_mrk_rx_bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(byte_diff)/tdiff; tbl_p->dl_mrk_rx_bps = RMNET_SHS_BYTE_TO_BIT(tbl_p->dl_mrk_rx_bps); } static void rmnet_shs_wq_refresh_total_stats(void) { struct rmnet_shs_wq_rx_flow_s *tbl_p = &rmnet_shs_rx_flow_tbl; u64 pkt_diff, byte_diff, pps, bps; time_t tdiff; tdiff = rmnet_shs_wq_tnsec - tbl_p->l_epoch; pkt_diff = (tbl_p->rx_skbs - tbl_p->last_rx_skbs); byte_diff = tbl_p->rx_bytes - tbl_p->last_rx_bytes; pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(pkt_diff)/tdiff; bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(byte_diff)/tdiff; tbl_p->last_rx_bps = tbl_p->rx_bps; tbl_p->last_rx_pps = tbl_p->rx_pps; tbl_p->rx_bps = RMNET_SHS_BYTE_TO_BIT(bps); tbl_p->rx_pps = pps; tbl_p->l_epoch = rmnet_shs_wq_tnsec; tbl_p->last_rx_bytes = tbl_p->rx_bytes; tbl_p->last_rx_skbs = tbl_p->rx_skbs; trace_rmnet_shs_wq_high(RMNET_SHS_WQ_TOTAL_STATS, RMNET_SHS_WQ_TOTAL_STATS_UPDATE, tbl_p->rx_pps, tbl_p->dl_mrk_rx_pps, tbl_p->rx_bps, tbl_p->dl_mrk_rx_bps, NULL, NULL); } static void rmnet_shs_wq_refresh_cpu_stats(u16 cpu) { struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_p; time_t tdiff; u64 new_skbs, new_bytes; u64 last_rx_bps, last_rx_pps; u32 new_qhead; if (cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return; } cpu_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu]; new_skbs = cpu_p->rx_skbs - cpu_p->last_rx_skbs; new_qhead = rmnet_shs_get_cpu_qhead(cpu); if (cpu_p->qhead_start == 0) cpu_p->qhead_start = new_qhead; cpu_p->last_qhead = cpu_p->qhead; cpu_p->qhead = new_qhead; cpu_p->qhead_diff = cpu_p->qhead - cpu_p->last_qhead; cpu_p->qhead_total = cpu_p->qhead - cpu_p->qhead_start; if (rmnet_shs_cpu_node_tbl[cpu].wqprio) rmnet_shs_cpu_node_tbl[cpu].wqprio = (rmnet_shs_cpu_node_tbl[cpu].wqprio + 1) % (PRIO_BACKOFF); if (new_skbs == 0) { cpu_p->l_epoch = rmnet_shs_wq_tnsec; cpu_p->rx_bps = 0; cpu_p->rx_pps = 0; cpu_p->avg_pps = 0; if (rmnet_shs_userspace_connected) { rmnet_shs_wq_cpu_caps_list_add(&rmnet_shs_rx_flow_tbl, cpu_p, &cpu_caps); } rmnet_shs_wq_refresh_cpu_rates_debug(cpu, cpu_p); return; } tdiff = rmnet_shs_wq_tnsec - cpu_p->l_epoch; new_bytes = cpu_p->rx_bytes - cpu_p->last_rx_bytes; last_rx_bps = cpu_p->rx_bps; last_rx_pps = cpu_p->rx_pps; cpu_p->rx_pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(new_skbs)/tdiff; cpu_p->rx_bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(new_bytes)/tdiff; cpu_p->rx_bps = RMNET_SHS_BYTE_TO_BIT(cpu_p->rx_bps); cpu_p->avg_pps = rmnet_shs_wq_get_cpu_avg_pps(cpu); cpu_p->last_rx_bps = last_rx_bps; cpu_p->last_rx_pps = last_rx_pps; cpu_p->l_epoch = rmnet_shs_wq_tnsec; cpu_p->last_rx_skbs = cpu_p->rx_skbs; cpu_p->last_rx_bytes = cpu_p->rx_bytes; cpu_p->rx_bps_est = cpu_p->rx_bps; if (rmnet_shs_userspace_connected) { rmnet_shs_wq_cpu_caps_list_add(&rmnet_shs_rx_flow_tbl, cpu_p, &cpu_caps); } trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_UPDATE, cpu, cpu_p->flows, cpu_p->rx_pps, cpu_p->rx_bps, NULL, NULL); rmnet_shs_wq_refresh_cpu_rates_debug(cpu, cpu_p); } static void rmnet_shs_wq_refresh_all_cpu_stats(void) { u16 cpu; trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); for (cpu = 0; cpu < MAX_CPUS; cpu++) rmnet_shs_wq_refresh_cpu_stats(cpu); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_END, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } void rmnet_shs_wq_update_cpu_rx_tbl(struct rmnet_shs_wq_hstat_s *hstat_p) { struct rmnet_shs_wq_rx_flow_s *tbl_p = &rmnet_shs_rx_flow_tbl; struct rmnet_shs_skbn_s *node_p; u64 skb_diff, byte_diff; u16 cpu_num; if (!hstat_p) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } node_p = hstat_p->node; if (hstat_p->inactive_duration > 0) return; cpu_num = node_p->map_cpu; if (cpu_num >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_INVALID_CPU_ERR]++; return; } skb_diff = hstat_p->rx_skb - hstat_p->last_rx_skb; byte_diff = hstat_p->rx_bytes - hstat_p->last_rx_bytes; if (hstat_p->is_new_flow) { rmnet_shs_wq_cpu_list_add(hstat_p, &tbl_p->cpu_list[cpu_num].hstat_id); rm_err("SHS_FLOW: adding flow 0x%x on cpu[%d] " "pps: %llu | avg_pps %llu", hstat_p->hash, hstat_p->current_cpu, hstat_p->rx_pps, hstat_p->avg_pps); hstat_p->is_new_flow = 0; } /* check if the flow has switched to another CPU*/ if (cpu_num != hstat_p->current_cpu) { rm_err("SHS_FLOW: moving flow 0x%x on cpu[%d] to cpu[%d] " "pps: %llu | avg_pps %llu", hstat_p->hash, hstat_p->current_cpu, cpu_num, hstat_p->rx_pps, hstat_p->avg_pps); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_UPDATE_NEW_CPU, hstat_p->hash, hstat_p->current_cpu, cpu_num, 0xDEF, hstat_p, NULL); rmnet_shs_wq_cpu_list_move(hstat_p, &tbl_p->cpu_list[cpu_num].hstat_id); rmnet_shs_wq_inc_cpu_flow(cpu_num); rmnet_shs_wq_dec_cpu_flow(hstat_p->current_cpu); hstat_p->current_cpu = cpu_num; } /* Assuming that the data transfers after the last refresh * interval have happened with the newer CPU */ tbl_p->cpu_list[cpu_num].rx_skbs += skb_diff; tbl_p->cpu_list[cpu_num].rx_bytes += byte_diff; tbl_p->rx_skbs += skb_diff; tbl_p->rx_bytes += byte_diff; } void rmnet_shs_wq_chng_suggested_cpu(u16 old_cpu, u16 new_cpu, struct rmnet_shs_wq_ep_s *ep) { struct rmnet_shs_skbn_s *node_p; struct rmnet_shs_wq_hstat_s *hstat_p; u16 bkt; hash_for_each(RMNET_SHS_HT, bkt, node_p, list) { if (!node_p) continue; if (!node_p->hstats) continue; hstat_p = node_p->hstats; if ((hstat_p->suggested_cpu == old_cpu) && (node_p->dev == ep->ep)) { trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_SUGGEST_NEW_CPU, hstat_p->hash, hstat_p->suggested_cpu, new_cpu, 0xDEF, hstat_p, NULL); node_p->hstats->suggested_cpu = new_cpu; } } } /* Increment the per-flow counter for suggestion type */ static void rmnet_shs_wq_inc_sugg_type(u32 sugg_type, struct rmnet_shs_wq_hstat_s *hstat_p) { if (sugg_type >= RMNET_SHS_WQ_SUGG_MAX || hstat_p == NULL) return; hstat_p->rmnet_shs_wq_suggs[sugg_type] += 1; } /* Change suggested cpu, return 1 if suggestion was made, 0 otherwise */ static int rmnet_shs_wq_chng_flow_cpu(u16 old_cpu, u16 new_cpu, struct rmnet_shs_wq_ep_s *ep, u32 hash_to_move, u32 sugg_type) { struct rmnet_shs_skbn_s *node_p; struct rmnet_shs_wq_hstat_s *hstat_p; int rc = 0; u16 bkt; if (!ep) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return 0; } if (old_cpu >= MAX_CPUS || new_cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return 0; } hash_for_each(RMNET_SHS_HT, bkt, node_p, list) { if (!node_p) continue; if (!node_p->hstats) continue; hstat_p = node_p->hstats; if (hash_to_move != 0) { /* If hash_to_move is given, only move that flow, * otherwise move all the flows on that cpu */ if (hstat_p->hash != hash_to_move) continue; } rm_err("SHS_HT: >> sugg cpu %d | old cpu %d | new_cpu %d | " "map_cpu = %d | flow 0x%x", hstat_p->suggested_cpu, old_cpu, new_cpu, node_p->map_cpu, hash_to_move); if ((hstat_p->suggested_cpu == old_cpu) && (node_p->dev == ep->ep)) { trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_SUGGEST_NEW_CPU, hstat_p->hash, hstat_p->suggested_cpu, new_cpu, 0xDEF, hstat_p, NULL); node_p->hstats->suggested_cpu = new_cpu; rmnet_shs_wq_inc_sugg_type(sugg_type, hstat_p); if (hash_to_move) { /* Stop after moving one flow */ rm_err("SHS_CHNG: moving single flow: flow 0x%x " "sugg_cpu changed from %d to %d", hstat_p->hash, old_cpu, node_p->hstats->suggested_cpu); return 1; } rm_err("SHS_CHNG: moving all flows: flow 0x%x " "sugg_cpu changed from %d to %d", hstat_p->hash, old_cpu, node_p->hstats->suggested_cpu); rc |= 1; } } return rc; } u64 rmnet_shs_wq_get_max_pps_among_cores(u32 core_msk) { int cpu_num; u64 max_pps = 0; struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl; for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { if (((1 << cpu_num) & core_msk) && (rx_flow_tbl_p->cpu_list[cpu_num].rx_pps > max_pps)) { max_pps = rx_flow_tbl_p->cpu_list[cpu_num].rx_pps; } } return max_pps; } /* Returns the least utilized core from a core mask * In order of priority * 1) Returns leftmost core with no flows (Fully Idle) * 2) Returns the core with least flows with no pps (Semi Idle) * 3) Returns the core with the least pps (Non-Idle) */ int rmnet_shs_wq_get_least_utilized_core(u16 core_msk) { struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl; struct rmnet_shs_wq_cpu_rx_pkt_q_s *list_p; u64 min_pps = U64_MAX; u32 min_flows = U32_MAX; int ret_val = -1; int semi_idle_ret = -1; int full_idle_ret = -1; int cpu_num = 0; u16 is_cpu_in_msk; for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { is_cpu_in_msk = (1 << cpu_num) & core_msk; if (!is_cpu_in_msk) continue; list_p = &rx_flow_tbl_p->cpu_list[cpu_num]; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_CURRENT_UTIL, cpu_num, list_p->rx_pps, min_pps, 0, NULL, NULL); /* When there are multiple free CPUs the first free CPU will * be returned */ if (list_p->flows == 0) { full_idle_ret = cpu_num; break; } /* When there are semi-idle CPUs the CPU w/ least flows will * be returned */ if (list_p->rx_pps == 0 && list_p->flows < min_flows) { min_flows = list_p->flows; semi_idle_ret = cpu_num; } /* Found a core that is processing even lower packets */ if (list_p->rx_pps <= min_pps) { min_pps = list_p->rx_pps; ret_val = cpu_num; } } if (full_idle_ret >= 0) ret_val = full_idle_ret; else if (semi_idle_ret >= 0) ret_val = semi_idle_ret; return ret_val; } u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, struct rmnet_shs_wq_ep_s *ep) { struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl; struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_list_p, *cur_cpu_list_p; u64 cpu_rx_pps, reqd_pps, cur_cpu_rx_pps; u64 pps_uthresh, pps_lthresh = 0; u16 cpu_to_move = current_cpu; u16 cpu_num; u8 is_core_in_msk; u32 cpu_to_move_util = 0; if (!ep) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return cpu_to_move; } cur_cpu_list_p = &rx_flow_tbl_p->cpu_list[current_cpu]; cur_cpu_rx_pps = cur_cpu_list_p->rx_pps; pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[current_cpu]; pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[current_cpu]; /* If we are already on a perf core and required pps is beyond * beyond the capacity that even perf cores aren't sufficient * there is nothing much we can do. So we will continue to let flows * process packets on same perf core */ if (!rmnet_shs_is_lpwr_cpu(current_cpu) && (cur_cpu_rx_pps > pps_lthresh)) { return cpu_to_move; } /* If a core (should only be lpwr was marked prio we don't touch it * for a few ticks and reset it afterwards */ if (rmnet_shs_cpu_node_tbl[current_cpu].wqprio) return current_cpu; for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { is_core_in_msk = ((1 << cpu_num) & ep->rps_config_msk); /* We are looking for a core that is configured and that * can handle traffic better than the current core */ if ((cpu_num == current_cpu) || (!is_core_in_msk) || !cpu_online(current_cpu)) continue; pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[cpu_num]; pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[cpu_num]; cpu_list_p = &rx_flow_tbl_p->cpu_list[cpu_num]; cpu_rx_pps = cpu_list_p->rx_pps; reqd_pps = cpu_rx_pps + cur_cpu_rx_pps; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND, current_cpu, cpu_num, reqd_pps, cpu_rx_pps, NULL, NULL); /* Return the most available valid CPU */ if ((reqd_pps > pps_lthresh) && (reqd_pps < pps_uthresh) && cpu_rx_pps <= cpu_to_move_util) { cpu_to_move = cpu_num; cpu_to_move_util = cpu_rx_pps; } } trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND, current_cpu, cpu_to_move, cur_cpu_rx_pps, rx_flow_tbl_p->cpu_list[cpu_to_move].rx_pps, NULL, NULL); return cpu_to_move; } void rmnet_shs_wq_find_cpu_and_move_flows(u16 cur_cpu) { struct rmnet_shs_wq_ep_s *ep = NULL; u16 new_cpu; list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (!ep->is_ep_active) continue; new_cpu = rmnet_shs_wq_find_cpu_to_move_flows(cur_cpu, ep); if (new_cpu != cur_cpu) rmnet_shs_wq_chng_suggested_cpu(cur_cpu, new_cpu, ep); } } /* Return 1 if we can move a flow to dest_cpu for this endpoint, * otherwise return 0. Basically check rps mask and cpu is online * Also check that dest cpu is not isolated */ int rmnet_shs_wq_check_cpu_move_for_ep(u16 current_cpu, u16 dest_cpu, struct rmnet_shs_wq_ep_s *ep) { u16 cpu_in_rps_mask = 0; if (!ep) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return 0; } if (current_cpu >= MAX_CPUS || dest_cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return 0; } cpu_in_rps_mask = (1 << dest_cpu) & ep->rps_config_msk; rm_err("SHS_MASK: cur cpu [%d] | dest_cpu [%d] | " "cpu isolation_mask = 0x%x | ep_rps_mask = 0x%x | " "cpu_online(dest) = %d cpu_in_rps_mask = %d | " "cpu isolated(dest) = %d", current_cpu, dest_cpu, __cpu_isolated_mask, ep->rps_config_msk, cpu_online(dest_cpu), cpu_in_rps_mask, cpu_isolated(dest_cpu)); /* We cannot move to dest cpu if the cur cpu is the same, * the dest cpu is offline, dest cpu is not in the rps mask, * or if the dest cpu is isolated */ if (current_cpu == dest_cpu || !cpu_online(dest_cpu) || !cpu_in_rps_mask || cpu_isolated(dest_cpu)) { return 0; } return 1; } /* rmnet_shs_wq_try_to_move_flow - try to make a flow suggestion * return 1 if flow move was suggested, otherwise return 0 */ int rmnet_shs_wq_try_to_move_flow(u16 cur_cpu, u16 dest_cpu, u32 hash_to_move, u32 sugg_type) { unsigned long flags; struct rmnet_shs_wq_ep_s *ep; if (cur_cpu >= MAX_CPUS || dest_cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return 0; } /* Traverse end-point list, check if cpu can be used, based * on it if is online, rps mask, isolation, etc. then make * suggestion to change the cpu for the flow by passing its hash */ spin_lock_irqsave(&rmnet_shs_ep_lock, flags); list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (!ep->is_ep_active) continue; if (!rmnet_shs_wq_check_cpu_move_for_ep(cur_cpu, dest_cpu, ep)) { rm_err("SHS_FDESC: >> Cannot move flow 0x%x on ep" " from cpu[%d] to cpu[%d]", hash_to_move, cur_cpu, dest_cpu); continue; } if (rmnet_shs_wq_chng_flow_cpu(cur_cpu, dest_cpu, ep, hash_to_move, sugg_type)) { rm_err("SHS_FDESC: >> flow 0x%x was suggested to" " move from cpu[%d] to cpu[%d] sugg_type [%d]", hash_to_move, cur_cpu, dest_cpu, sugg_type); spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return 1; } } spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return 0; } /* Change flow segmentation, return 1 if set, 0 otherwise */ int rmnet_shs_wq_set_flow_segmentation(u32 hash_to_set, u8 segs_per_skb) { struct rmnet_shs_skbn_s *node_p; struct rmnet_shs_wq_hstat_s *hstat_p; unsigned long ht_flags; u16 bkt; spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); hash_for_each(RMNET_SHS_HT, bkt, node_p, list) { if (!node_p) continue; if (!node_p->hstats) continue; hstat_p = node_p->hstats; if (hstat_p->hash != hash_to_set) continue; rm_err("SHS_HT: >> segmentation on hash 0x%x segs_per_skb %u", hash_to_set, segs_per_skb); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_SET_FLOW_SEGMENTATION, hstat_p->hash, segs_per_skb, 0xDEF, 0xDEF, hstat_p, NULL); node_p->hstats->segs_per_skb = segs_per_skb; spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); return 1; } spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); rm_err("SHS_HT: >> segmentation on hash 0x%x segs_per_skb %u not set - hash not found", hash_to_set, segs_per_skb); return 0; } /* Comparison function to sort gold flow loads - based on flow avg_pps * return -1 if a is before b, 1 if a is after b, 0 if equal */ int cmp_fn_flow_pps(void *priv, struct list_head *a, struct list_head *b) { struct rmnet_shs_wq_gold_flow_s *flow_a; struct rmnet_shs_wq_gold_flow_s *flow_b; if (!a || !b) return 0; flow_a = list_entry(a, struct rmnet_shs_wq_gold_flow_s, gflow_list); flow_b = list_entry(b, struct rmnet_shs_wq_gold_flow_s, gflow_list); if (flow_a->avg_pps > flow_b->avg_pps) return -1; else if (flow_a->avg_pps < flow_b->avg_pps) return 1; return 0; } /* Comparison function to sort cpu capacities - based on cpu avg_pps capacity * return -1 if a is before b, 1 if a is after b, 0 if equal */ int cmp_fn_cpu_pps(void *priv, struct list_head *a, struct list_head *b) { struct rmnet_shs_wq_cpu_cap_s *cpu_a; struct rmnet_shs_wq_cpu_cap_s *cpu_b; if (!a || !b) return 0; cpu_a = list_entry(a, struct rmnet_shs_wq_cpu_cap_s, cpu_cap_list); cpu_b = list_entry(b, struct rmnet_shs_wq_cpu_cap_s, cpu_cap_list); if (cpu_a->avg_pps_capacity > cpu_b->avg_pps_capacity) return -1; else if (cpu_a->avg_pps_capacity < cpu_b->avg_pps_capacity) return 1; return 0; } /* Prints cpu stats and flows to dmesg for debugging */ void rmnet_shs_wq_debug_print_flows(void) { struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl; struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_node; struct rmnet_shs_wq_hstat_s *hnode; int flows, i; u16 cpu_num = 0; if (!RMNET_SHS_DEBUG) return; for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { cpu_node = &rx_flow_tbl_p->cpu_list[cpu_num]; flows = rx_flow_tbl_p->cpu_list[cpu_num].flows; rm_err("SHS_CPU: cpu[%d]: flows=%d pps=%llu bps=%llu " "qhead_diff %u qhead_total = %u qhead_start = %u " "qhead = %u qhead_last = %u isolated = %d ", cpu_num, flows, cpu_node->rx_pps, cpu_node->rx_bps, cpu_node->qhead_diff, cpu_node->qhead_total, cpu_node->qhead_start, cpu_node->qhead, cpu_node->last_qhead, cpu_isolated(cpu_num)); list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (!hnode) continue; if (hnode->in_use == 0) continue; if (hnode->node) { if (hnode->current_cpu == cpu_num) rm_err("SHS_CPU: > flow 0x%x " "with pps %llu avg_pps %llu rx_bps %llu ", hnode->hash, hnode->rx_pps, hnode->avg_pps, hnode->rx_bps); } } /* loop per flow */ for (i = 0; i < 3 - flows; i++) { rm_err("%s", "SHS_CPU: > "); } } /* loop per cpu */ } /* Prints the sorted gold flow list to dmesg */ void rmnet_shs_wq_debug_print_sorted_gold_flows(struct list_head *gold_flows) { struct rmnet_shs_wq_gold_flow_s *gflow_node; if (!RMNET_SHS_DEBUG) return; if (!gold_flows) { rm_err("%s", "SHS_GDMA: Gold Flows List is NULL"); return; } rm_err("%s", "SHS_GDMA: List of sorted gold flows:"); list_for_each_entry(gflow_node, gold_flows, gflow_list) { if (!gflow_node) continue; rm_err("SHS_GDMA: > flow 0x%x with pps %llu on cpu[%d]", gflow_node->hash, gflow_node->rx_pps, gflow_node->cpu_num); } } /* Userspace evaluation. we send userspace the response to the sync message * after we update shared memory. shsusr will send a netlink message if * flows should be moved around. */ void rmnet_shs_wq_eval_cpus_caps_and_flows(struct list_head *cpu_caps, struct list_head *gold_flows, struct list_head *ss_flows) { if (!cpu_caps || !gold_flows || !ss_flows) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++; return; } list_sort(NULL, cpu_caps, &cmp_fn_cpu_pps); list_sort(NULL, gold_flows, &cmp_fn_flow_pps); rmnet_shs_wq_mem_update_cached_cpu_caps(cpu_caps); rmnet_shs_wq_mem_update_cached_sorted_gold_flows(gold_flows); rmnet_shs_wq_mem_update_cached_sorted_ss_flows(ss_flows); rmnet_shs_wq_mem_update_cached_netdevs(); rmnet_shs_genl_send_int_to_userspace_no_info(RMNET_SHS_SYNC_RESP_INT); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_SHSUSR, RMNET_SHS_WQ_SHSUSR_SYNC_END, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } /* Default wq evaluation logic, use this if rmnet_shs_userspace_connected is 0 */ void rmnet_shs_wq_eval_suggested_cpu(void) { struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl; struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_list_p; u64 cpu_curr_pps, cpu_last_pps, last_avg_pps; u64 moving_avg_pps, avg_pps; u64 pps_uthresh, pps_lthresh = 0; u16 cpu_num, new_weight, old_weight; int flows; for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { flows = rx_flow_tbl_p->cpu_list[cpu_num].flows; /* Nothing to evaluate if there is no traffic on this cpu */ if (flows <= 0) continue; cpu_list_p = &rx_flow_tbl_p->cpu_list[cpu_num]; cpu_curr_pps = cpu_list_p->rx_pps; cpu_last_pps = cpu_list_p->last_rx_pps; last_avg_pps = cpu_list_p->avg_pps; pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[cpu_num]; pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[cpu_num]; /* Often when we decide to switch from a small cluster core, * it is because of the heavy traffic on that core. In such * circumstances, we want to switch to a big cluster * core as soon as possible. Therefore, we will provide a * greater weightage to the most recent sample compared to * the previous samples. * * On the other hand, when a flow which is on a big cluster * cpu suddenly starts to receive low traffic we move to a * small cluster core after observing low traffic for some * more samples. This approach avoids switching back and forth * to small cluster cpus due to momentary decrease in data * traffic. */ if (rmnet_shs_is_lpwr_cpu(cpu_num)) { new_weight = rmnet_shs_wq_tuning; old_weight = 100 - rmnet_shs_wq_tuning; } else { old_weight = rmnet_shs_wq_tuning; new_weight = 100 - rmnet_shs_wq_tuning; } /*computing weighted average*/ moving_avg_pps = (cpu_last_pps + last_avg_pps) / 2; avg_pps = ((new_weight * cpu_curr_pps) + (old_weight * moving_avg_pps)) / (new_weight + old_weight); cpu_list_p->avg_pps = avg_pps; trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_EVAL_CPU, cpu_num, cpu_curr_pps, cpu_last_pps, avg_pps, NULL, NULL); if ((avg_pps > pps_uthresh) || ((avg_pps < pps_lthresh) && (cpu_curr_pps < pps_lthresh))) rmnet_shs_wq_find_cpu_and_move_flows(cpu_num); } } void rmnet_shs_wq_refresh_new_flow_list_per_ep(struct rmnet_shs_wq_ep_s *ep) { int lo_core; int hi_core; u16 rps_msk; u16 lo_msk; u16 hi_msk; u8 lo_core_idx = 0; u8 hi_core_idx = 0; if (!ep) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return; } rps_msk = ep->rps_config_msk; lo_msk = ep->default_core_msk; hi_msk = ep->pri_core_msk; memset(ep->new_lo_core, -1, sizeof(*ep->new_lo_core) * MAX_CPUS); memset(ep->new_hi_core, -1, sizeof(*ep->new_hi_core) * MAX_CPUS); do { lo_core = rmnet_shs_wq_get_least_utilized_core(lo_msk); if (lo_core >= 0) { ep->new_lo_core[lo_core_idx] = lo_core; lo_msk = lo_msk & ~(1 << lo_core); lo_core_idx++; } else { break; } } while (lo_msk != 0); trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_NEW_FLOW_LIST_LO, ep->new_lo_core[0], ep->new_lo_core[1], ep->new_lo_core[2], ep->new_lo_max, ep, NULL); do { hi_core = rmnet_shs_wq_get_least_utilized_core(hi_msk); if (hi_core >= 0) { ep->new_hi_core[hi_core_idx] = hi_core; hi_msk = hi_msk & ~(1 << hi_core); hi_core_idx++; } else break; } while (hi_msk != 0); ep->new_lo_max = lo_core_idx; ep->new_hi_max = hi_core_idx; ep->new_lo_idx = 0; ep->new_hi_idx = 0; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_NEW_FLOW_LIST_HI, ep->new_hi_core[0], ep->new_hi_core[1], ep->new_hi_core[2], ep->new_hi_max, ep, NULL); return; } void rmnet_shs_wq_refresh_new_flow_list(void) { struct rmnet_shs_wq_ep_s *ep = NULL; list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (!ep->is_ep_active) continue; rmnet_shs_wq_refresh_new_flow_list_per_ep(ep); } } /* Return Invalid core if only pri core available*/ int rmnet_shs_wq_get_lpwr_cpu_new_flow(struct net_device *dev) { u8 lo_idx; u8 lo_max; int cpu_assigned = -1; u8 is_match_found = 0; struct rmnet_shs_wq_ep_s *ep = NULL; unsigned long flags; if (!dev) { rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++; return cpu_assigned; } spin_lock_irqsave(&rmnet_shs_ep_lock, flags); list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (!ep->is_ep_active) continue; if (ep->ep == dev) { is_match_found = 1; break; } } if (!is_match_found) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return cpu_assigned; } lo_idx = ep->new_lo_idx; lo_max = ep->new_lo_max; while (lo_idx < lo_max) { if (ep->new_lo_core[lo_idx] >= 0) { cpu_assigned = ep->new_lo_core[lo_idx]; break; } lo_idx++; } /* Increment CPU assignment idx to be ready for next flow assignment*/ if ((cpu_assigned >= 0) || ((ep->new_lo_idx + 1) >= ep->new_lo_max)) ep->new_lo_idx = ((ep->new_lo_idx + 1) % ep->new_lo_max); spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return cpu_assigned; } int rmnet_shs_wq_get_perf_cpu_new_flow(struct net_device *dev) { struct rmnet_shs_wq_ep_s *ep = NULL; int cpu_assigned = -1; u8 hi_idx; u8 hi_max; u8 is_match_found = 0; unsigned long flags; if (!dev) { rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++; return cpu_assigned; } spin_lock_irqsave(&rmnet_shs_ep_lock, flags); list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (!ep->is_ep_active) continue; if (ep->ep == dev) { is_match_found = 1; break; } } if (!is_match_found) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return cpu_assigned; } hi_idx = ep->new_hi_idx; hi_max = ep->new_hi_max; while (hi_idx < hi_max) { if (ep->new_hi_core[hi_idx] >= 0) { cpu_assigned = ep->new_hi_core[hi_idx]; break; } hi_idx++; } /* Increment CPU assignment idx to be ready for next flow assignment*/ if (cpu_assigned >= 0) ep->new_hi_idx = ((hi_idx + 1) % hi_max); spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return cpu_assigned; } static int rmnet_shs_wq_time_check(time_t time, int num_flows) { int ret = false; if (time > rmnet_shs_max_flow_inactivity_sec) ret = true; else if (num_flows > FLOW_LIMIT2 && time > INACTIVE_TSEC2) ret = true; else if (num_flows > FLOW_LIMIT1 && time > INACTIVE_TSEC1) ret = true; return ret; } void rmnet_shs_wq_cleanup_hash_tbl(u8 force_clean) { struct rmnet_shs_skbn_s *node_p = NULL; time_t tns2s; unsigned long ht_flags; struct rmnet_shs_wq_hstat_s *hnode = NULL; struct list_head *ptr = NULL, *next = NULL; list_for_each_safe(ptr, next, &rmnet_shs_wq_hstat_tbl) { hnode = list_entry(ptr, struct rmnet_shs_wq_hstat_s, hstat_node_id); if (hnode == NULL) continue; if (hnode->node == NULL) continue; node_p = hnode->node; tns2s = RMNET_SHS_NSEC_TO_SEC(hnode->inactive_duration); /* Flows are cleanup from book keeping faster if * there are a lot of active flows already in memory */ if (rmnet_shs_wq_time_check(tns2s, rmnet_shs_cfg.num_flows) || force_clean) { trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_FLOW_INACTIVE_TIMEOUT, node_p->hash, tns2s, 0xDEF, 0xDEF, node_p, hnode); spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); rmnet_shs_clear_node(node_p, RMNET_WQ_CTXT); rmnet_shs_wq_dec_cpu_flow(hnode->current_cpu); if (node_p) { rmnet_shs_cpu_node_remove(node_p); hash_del_rcu(&node_p->list); kfree(node_p); } rm_err("SHS_FLOW: removing flow 0x%x on cpu[%d] " "pps: %llu avg_pps: %llu", hnode->hash, hnode->current_cpu, hnode->rx_pps, hnode->avg_pps); rmnet_shs_wq_cpu_list_remove(hnode); if (hnode->is_perm == 0 || force_clean) { rmnet_shs_wq_hstat_tbl_remove(hnode); kfree(hnode); } else { rmnet_shs_wq_hstat_reset_node(hnode); } rmnet_shs_cfg.num_flows--; spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); } } } void rmnet_shs_wq_update_ep_rps_msk(struct rmnet_shs_wq_ep_s *ep) { struct rps_map *map; u8 len = 0; if (!ep || !ep->ep ) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return; } rcu_read_lock(); if (!ep->ep) { pr_info(" rmnet_shs invalid state %p", ep->ep); rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return; } map = rcu_dereference(ep->ep->_rx->rps_map); ep->rps_config_msk = 0; if (map != NULL) { for (len = 0; len < map->len; len++) ep->rps_config_msk |= (1 << map->cpus[len]); } rcu_read_unlock(); ep->default_core_msk = ep->rps_config_msk & 0x0F; ep->pri_core_msk = ep->rps_config_msk & 0xF0; } void rmnet_shs_wq_reset_ep_active(struct net_device *dev) { struct rmnet_shs_wq_ep_s *ep = NULL; struct rmnet_shs_wq_ep_s *tmp = NULL; unsigned long flags; if (!dev) { rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++; return; } spin_lock_irqsave(&rmnet_shs_ep_lock, flags); list_for_each_entry_safe(ep, tmp, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (ep->ep == dev){ ep->is_ep_active = 0; rmnet_shs_wq_ep_tbl_remove(ep); kfree(ep); break; } } spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); } void rmnet_shs_wq_set_ep_active(struct net_device *dev) { struct rmnet_shs_wq_ep_s *ep = NULL; unsigned long flags; if (!dev) { rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++; return; } spin_lock_irqsave(&rmnet_shs_ep_lock, flags); ep = kzalloc(sizeof(*ep), GFP_ATOMIC); if (!ep) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_EP_TBL_ERR]++; spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); return; } ep->ep = dev; ep->is_ep_active = 1; INIT_LIST_HEAD(&ep->ep_list_id); rmnet_shs_wq_update_ep_rps_msk(ep); rmnet_shs_wq_ep_tbl_add(ep); spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); } void rmnet_shs_wq_refresh_ep_masks(void) { struct rmnet_shs_wq_ep_s *ep = NULL; list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep) continue; if (!ep->is_ep_active) continue; rmnet_shs_wq_update_ep_rps_msk(ep); /* These tput totals get re-added as we go through each flow */ ep->udp_rx_bps = 0; ep->tcp_rx_bps = 0; } } void rmnet_shs_update_cfg_mask(void) { /* Start with most avaible mask all eps could share*/ u8 mask = UPDATE_MASK; u8 rps_enabled = 0; struct rmnet_shs_wq_ep_s *ep; list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { if (!ep->is_ep_active) continue; /* Bitwise and to get common mask from non-null masks. * VNDs with different mask will have UNDEFINED behavior */ if (ep->rps_config_msk) { mask &= ep->rps_config_msk; rps_enabled = 1; } } if (!rps_enabled) { rmnet_shs_cfg.map_mask = 0; rmnet_shs_cfg.map_len = 0; return; } else if (rmnet_shs_cfg.map_mask != mask) { rmnet_shs_cfg.map_mask = mask; rmnet_shs_cfg.map_len = rmnet_shs_get_mask_len(mask); } } noinline void rmnet_shs_wq_filter(void) { int cpu, cur_cpu; int temp; struct rmnet_shs_wq_hstat_s *hnode = NULL; for (cpu = 0; cpu < MAX_CPUS; cpu++) { rmnet_shs_cpu_rx_filter_flows[cpu] = 0; rmnet_shs_cpu_node_tbl[cpu].seg = 0; } /* Filter out flows with low pkt count and * mark CPUS with slowstart flows */ list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (hnode->in_use == 0) continue; if (hnode->avg_pps > RMNET_SHS_FILTER_FLOW_RATE && hnode->rx_skb > RMNET_SHS_FILTER_PKT_LIMIT) if (hnode->current_cpu < MAX_CPUS){ temp = hnode->current_cpu; rmnet_shs_cpu_rx_filter_flows[temp]++; } cur_cpu = hnode->current_cpu; if (cur_cpu >= MAX_CPUS || cur_cpu < 0) { continue; } if (hnode->segs_per_skb > 0) { rmnet_shs_cpu_node_tbl[cur_cpu].seg++; } } } void rmnet_shs_wq_update_stats(void) { struct timespec time; struct rmnet_shs_wq_hstat_s *hnode = NULL; (void) getnstimeofday(&time); rmnet_shs_wq_tnsec = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) + time.tv_nsec; rmnet_shs_wq_refresh_ep_masks(); rmnet_shs_update_cfg_mask(); list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (!hnode) continue; if (hnode->in_use == 0) continue; if (hnode->node) { rmnet_shs_wq_update_hash_stats(hnode); rmnet_shs_wq_update_cpu_rx_tbl(hnode); if (rmnet_shs_userspace_connected) { if (!rmnet_shs_is_lpwr_cpu(hnode->current_cpu)) { /* Add golds flows to list */ rmnet_shs_wq_gflow_list_add(hnode, &gflows); } if (hnode->skb_tport_proto == IPPROTO_TCP) { rmnet_shs_wq_ssflow_list_add(hnode, &ssflows); } } else { /* Disable segmentation if userspace gets disconnected connected */ hnode->node->hstats->segs_per_skb = 0; } } } rmnet_shs_wq_refresh_all_cpu_stats(); rmnet_shs_wq_refresh_total_stats(); rmnet_shs_wq_refresh_dl_mrkr_stats(); if (rmnet_shs_userspace_connected) { rm_err("%s", "SHS_UPDATE: Userspace connected, relying on userspace evaluation"); rmnet_shs_wq_eval_cpus_caps_and_flows(&cpu_caps, &gflows, &ssflows); rmnet_shs_wq_cleanup_gold_flow_list(&gflows); rmnet_shs_wq_cleanup_ss_flow_list(&ssflows); rmnet_shs_wq_cleanup_cpu_caps_list(&cpu_caps); } else { rm_err("%s", "SHS_UPDATE: shs userspace not connected, using default logic"); rmnet_shs_wq_eval_suggested_cpu(); } rmnet_shs_wq_refresh_new_flow_list(); rmnet_shs_wq_filter(); } void rmnet_shs_wq_process_wq(struct work_struct *work) { unsigned long flags; unsigned long jiffies; trace_rmnet_shs_wq_high(RMNET_SHS_WQ_PROCESS_WQ, RMNET_SHS_WQ_PROCESS_WQ_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); spin_lock_irqsave(&rmnet_shs_ep_lock, flags); rmnet_shs_wq_update_stats(); spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags); /*Invoke after both the locks are released*/ rmnet_shs_wq_cleanup_hash_tbl(PERIODIC_CLEAN); rmnet_shs_wq_debug_print_flows(); jiffies = msecs_to_jiffies(rmnet_shs_wq_interval_ms); queue_delayed_work(rmnet_shs_wq, &rmnet_shs_delayed_wq->wq, jiffies); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_PROCESS_WQ, RMNET_SHS_WQ_PROCESS_WQ_END, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } void rmnet_shs_wq_clean_ep_tbl(void) { struct rmnet_shs_wq_ep_s *ep = NULL; struct list_head *ptr = NULL, *next = NULL; list_for_each_safe(ptr, next, &rmnet_shs_wq_ep_tbl) { ep = list_entry(ptr, struct rmnet_shs_wq_ep_s, ep_list_id); if (!ep) continue; trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_CLEANUP, 0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL); rmnet_shs_wq_ep_tbl_remove(ep); kfree(ep); } } void rmnet_shs_wq_exit(void) { /*If Wq is not initialized, nothing to cleanup */ if (!rmnet_shs_wq || !rmnet_shs_delayed_wq) return; rmnet_shs_wq_mem_deinit(); rmnet_shs_genl_send_int_to_userspace_no_info(RMNET_SHS_SYNC_WQ_EXIT); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EXIT, RMNET_SHS_WQ_EXIT_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); cancel_delayed_work_sync(&rmnet_shs_delayed_wq->wq); drain_workqueue(rmnet_shs_wq); destroy_workqueue(rmnet_shs_wq); kfree(rmnet_shs_delayed_wq); rmnet_shs_delayed_wq = NULL; rmnet_shs_wq = NULL; rmnet_shs_wq_cleanup_hash_tbl(FORCE_CLEAN); rmnet_shs_wq_clean_ep_tbl(); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EXIT, RMNET_SHS_WQ_EXIT_END, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } void rmnet_shs_wq_init_cpu_rx_flow_tbl(void) { u8 cpu_num; struct rmnet_shs_wq_cpu_rx_pkt_q_s *rx_flow_tbl_p; for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_INIT, cpu_num, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu_num]; INIT_LIST_HEAD(&rx_flow_tbl_p->hstat_id); rx_flow_tbl_p->cpu_num = cpu_num; } } void rmnet_shs_wq_pause(void) { int cpu; if (rmnet_shs_wq && rmnet_shs_delayed_wq) cancel_delayed_work_sync(&rmnet_shs_delayed_wq->wq); for (cpu = 0; cpu < MAX_CPUS; cpu++) rmnet_shs_cpu_rx_filter_flows[cpu] = 0; } void rmnet_shs_wq_restart(void) { if (rmnet_shs_wq && rmnet_shs_delayed_wq) queue_delayed_work(rmnet_shs_wq, &rmnet_shs_delayed_wq->wq, 0); } void rmnet_shs_wq_init(struct net_device *dev) { /*If the workqueue is already initialized we should not be *initializing again */ if (rmnet_shs_wq) return; if (!dev) { rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++; return; } if( rmnet_shs_wq_mem_init() ) rmnet_shs_wq_genl_deinit(); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_INIT, RMNET_SHS_WQ_INIT_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); rmnet_shs_wq = alloc_workqueue("rmnet_shs_wq", WQ_CPU_INTENSIVE, 1); if (!rmnet_shs_wq) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_WQ_ERR]++; return; } rmnet_shs_delayed_wq = kmalloc(sizeof(struct rmnet_shs_delay_wq_s), GFP_ATOMIC); if (!rmnet_shs_delayed_wq) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_DEL_WQ_ERR]++; rmnet_shs_wq_exit(); return; } /*All hstat nodes allocated during Wq init will be held for ever*/ rmnet_shs_wq_hstat_alloc_nodes(RMNET_SHS_MIN_HSTAT_NODES_REQD, 1); rmnet_shs_wq_init_cpu_rx_flow_tbl(); INIT_DEFERRABLE_WORK(&rmnet_shs_delayed_wq->wq, rmnet_shs_wq_process_wq); trace_rmnet_shs_wq_high(RMNET_SHS_WQ_INIT, RMNET_SHS_WQ_INIT_END, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } int rmnet_shs_wq_get_num_cpu_flows(u16 cpu) { int flows = -1; if (cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_INVALID_CPU_ERR]++; return flows; } flows = rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_GET_CPU_FLOW, cpu, flows, 0xDEF, 0xDEF, NULL, NULL); return flows; } int rmnet_shs_wq_get_max_flows_per_core(void) { u16 cpu; int max_flows = -1; int cpu_flows; for (cpu = 0; cpu < MAX_CPUS; cpu++) { cpu_flows = rmnet_shs_wq_get_num_cpu_flows(cpu); if (cpu_flows > max_flows) max_flows = cpu_flows; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_GET_MAX_CPU_FLOW, cpu, cpu_flows, max_flows, 0xDEF, NULL, NULL); } return max_flows; } int rmnet_shs_wq_get_max_flows_per_cluster(u16 cpu) { u32 big_cluster_mask = 1<<4; u32 core_mask = 1; u16 start_core = 0; u16 end_core = 4; int max_flows = -1; int cpu_flows; if (cpu > MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_INVALID_CPU_ERR]++; return max_flows; } core_mask <<= cpu; if (core_mask >= big_cluster_mask) { start_core = 4; end_core = MAX_CPUS; } for (start_core; start_core < end_core; start_core++) { cpu_flows = rmnet_shs_wq_get_num_cpu_flows(start_core); if (cpu_flows > max_flows) max_flows = cpu_flows; } trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_MAX_FLOW_IN_CLUSTER, start_core, end_core, cpu, max_flows, NULL, NULL); return max_flows; } void rmnet_shs_wq_inc_cpu_flow(u16 cpu) { if (cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return; } rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows++; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_INC_CPU_FLOW, cpu, rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows, 0xDEF, 0xDEF, NULL, NULL); } void rmnet_shs_wq_dec_cpu_flow(u16 cpu) { if (cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return; } if (rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows > 0) rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows--; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, RMNET_SHS_WQ_CPU_STATS_DEC_CPU_FLOW, cpu, rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows, 0xDEF, 0xDEF, NULL, NULL); } u64 rmnet_shs_wq_get_max_allowed_pps(u16 cpu) { if (cpu >= MAX_CPUS) { rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++; return 0; } return rmnet_shs_cpu_rx_max_pps_thresh[cpu]; } void rmnet_shs_wq_ep_lock_bh(void) { spin_lock_bh(&rmnet_shs_ep_lock); } void rmnet_shs_wq_ep_unlock_bh(void) { spin_unlock_bh(&rmnet_shs_ep_lock); }