132 	if (rd->rp->priv->type == RAPL_IF_TPMI) {  in get_pl_lock_prim()
149 	if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))  in get_pl_lock_prim()
160 		if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)  in get_pl_prim()
174 		if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)  in get_pl_prim()
217 static struct rapl_defaults *get_defaults(struct rapl_package *rp)  in get_defaults()  argument
219 	return rp->priv->defaults;  in get_defaults()
250 static void rapl_init_domains(struct rapl_package *rp);
265 static void package_power_limit_irq_save(struct rapl_package *rp);
311 	struct rapl_package *rp = rd->rp;  in release_zone()  local
318 		rp->domains = NULL;  in release_zone()
340 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in set_domain_enable()
439 	struct rapl_package *rp;  in set_power_limit()  local
446 	rp = rd->rp;  in set_power_limit()
450 		package_power_limit_irq_save(rp);  in set_power_limit()
562 static int get_rid(struct rapl_package *rp)  in get_rid()  argument
564 	return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id;  in get_rid()
568 static void rapl_init_domains(struct rapl_package *rp)  in rapl_init_domains()  argument
572 	struct rapl_domain *rd = rp->domains;  in rapl_init_domains()
575 		unsigned int mask = rp->domain_map & (1 << i);  in rapl_init_domains()
581 		rd->rp = rp;  in rapl_init_domains()
583 		if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {  in rapl_init_domains()
585 				rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) :  in rapl_init_domains()
586 				rp->id);  in rapl_init_domains()
595 		rp->priv->limits[i] |= BIT(POWER_LIMIT1);  in rapl_init_domains()
598 			if (rp->priv->limits[i] & BIT(t))  in rapl_init_domains()
603 			rd->regs[j] = rp->priv->regs[i][j];  in rapl_init_domains()
613 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in rapl_unit_xlate()
737 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)  in get_rpi()  argument
739 	struct rapl_primitive_info *rpi = rp->priv->rpi;  in get_rpi()
747 static int rapl_config(struct rapl_package *rp)  in rapl_config()  argument
749 	switch (rp->priv->type) {  in rapl_config()
753 		rp->priv->defaults = (void *)defaults_msr;  in rapl_config()
754 		rp->priv->rpi = (void *)rpi_msr;  in rapl_config()
757 		rp->priv->defaults = (void *)&defaults_tpmi;  in rapl_config()
758 		rp->priv->rpi = (void *)rpi_tpmi;  in rapl_config()
765 	if (!rp->priv->defaults || !rp->priv->rpi)  in rapl_config()
774 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in prim_fixups()
818 	struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);  in rapl_read_data_raw()
836 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_read_data_raw()
837 		pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name);  in rapl_read_data_raw()
857 	struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);  in rapl_write_data_raw()
875 	ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra);  in rapl_write_data_raw()
901 		pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]);  in rapl_write_pl_data()
925 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_core()
927 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_core()
941 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_core()
953 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_atom()
955 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_atom()
969 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_atom()
977 	struct rapl_package *rp = (struct rapl_package *)info;  in power_limit_irq_save_cpu()  local
981 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {  in power_limit_irq_save_cpu()
982 		rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;  in power_limit_irq_save_cpu()
983 		rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;  in power_limit_irq_save_cpu()
999 static void package_power_limit_irq_save(struct rapl_package *rp)  in package_power_limit_irq_save()  argument
1001 	if (rp->lead_cpu < 0)  in package_power_limit_irq_save()
1007 	smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);  in package_power_limit_irq_save()
1014 static void package_power_limit_irq_restore(struct rapl_package *rp)  in package_power_limit_irq_restore()  argument
1018 	if (rp->lead_cpu < 0)  in package_power_limit_irq_restore()
1025 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))  in package_power_limit_irq_restore()
1030 	if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)  in package_power_limit_irq_restore()
1056 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in set_floor_freq_atom()
1140 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_tpmi()
1142 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_tpmi()
1156 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_tpmi()
1290 static void rapl_update_domain_data(struct rapl_package *rp)  in rapl_update_domain_data()  argument
1295 	for (dmn = 0; dmn < rp->nr_domains; dmn++) {  in rapl_update_domain_data()
1296 		pr_debug("update %s domain %s data\n", rp->name,  in rapl_update_domain_data()
1297 			 rp->domains[dmn].name);  in rapl_update_domain_data()
1300 			struct rapl_primitive_info *rpi = get_rpi(rp, prim);  in rapl_update_domain_data()
1302 			if (!rapl_read_data_raw(&rp->domains[dmn], prim,  in rapl_update_domain_data()
1304 				rp->domains[dmn].rdd.primitives[prim] = val;  in rapl_update_domain_data()
1310 static int rapl_package_register_powercap(struct rapl_package *rp)  in rapl_package_register_powercap()  argument
1317 	rapl_update_domain_data(rp);  in rapl_package_register_powercap()
1320 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_package_register_powercap()
1323 			pr_debug("register package domain %s\n", rp->name);  in rapl_package_register_powercap()
1325 					    rp->priv->control_type, rp->name,  in rapl_package_register_powercap()
1330 					 rp->name);  in rapl_package_register_powercap()
1334 			rp->power_zone = power_zone;  in rapl_package_register_powercap()
1344 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_package_register_powercap()
1345 		struct powercap_zone *parent = rp->power_zone;  in rapl_package_register_powercap()
1354 						    rp->priv->control_type,  in rapl_package_register_powercap()
1361 				 rp->name, rd->name);  in rapl_package_register_powercap()
1373 	while (--rd >= rp->domains) {  in rapl_package_register_powercap()
1374 		pr_debug("unregister %s domain %s\n", rp->name, rd->name);  in rapl_package_register_powercap()
1375 		powercap_unregister_zone(rp->priv->control_type,  in rapl_package_register_powercap()
1382 static int rapl_check_domain(int domain, struct rapl_package *rp)  in rapl_check_domain()  argument
1392 		ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];  in rapl_check_domain()
1403 	if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value)  in rapl_check_domain()
1416 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in rapl_get_domain_unit()
1420 		if (!rd->rp->priv->reg_unit.val) {  in rapl_get_domain_unit()
1424 		rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit;  in rapl_get_domain_unit()
1462 					rd->rp->name, rd->name, pl_names[i]);  in rapl_detect_powerlimit()
1474 static int rapl_detect_domains(struct rapl_package *rp)  in rapl_detect_domains()  argument
1481 		if (!rapl_check_domain(i, rp)) {  in rapl_detect_domains()
1482 			rp->domain_map |= 1 << i;  in rapl_detect_domains()
1486 	rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);  in rapl_detect_domains()
1487 	if (!rp->nr_domains) {  in rapl_detect_domains()
1488 		pr_debug("no valid rapl domains found in %s\n", rp->name);  in rapl_detect_domains()
1491 	pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);  in rapl_detect_domains()
1493 	rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain),  in rapl_detect_domains()
1495 	if (!rp->domains)  in rapl_detect_domains()
1498 	rapl_init_domains(rp);  in rapl_detect_domains()
1500 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_detect_domains()
1509 void rapl_remove_package_cpuslocked(struct rapl_package *rp)  in rapl_remove_package_cpuslocked()  argument
1513 	package_power_limit_irq_restore(rp);  in rapl_remove_package_cpuslocked()
1515 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_remove_package_cpuslocked()
1528 			 rp->name, rd->name);  in rapl_remove_package_cpuslocked()
1529 		powercap_unregister_zone(rp->priv->control_type,  in rapl_remove_package_cpuslocked()
1533 	powercap_unregister_zone(rp->priv->control_type,  in rapl_remove_package_cpuslocked()
1535 	list_del(&rp->plist);  in rapl_remove_package_cpuslocked()
1536 	kfree(rp);  in rapl_remove_package_cpuslocked()
1540 void rapl_remove_package(struct rapl_package *rp)  in rapl_remove_package()  argument
1543 	rapl_remove_package_cpuslocked(rp);  in rapl_remove_package()
1551 	struct rapl_package *rp;  in rapl_find_package_domain_cpuslocked()  local
1559 	list_for_each_entry(rp, &rapl_packages, plist) {  in rapl_find_package_domain_cpuslocked()
1560 		if (rp->id == uid  in rapl_find_package_domain_cpuslocked()
1561 		    && rp->priv->control_type == priv->control_type)  in rapl_find_package_domain_cpuslocked()
1562 			return rp;  in rapl_find_package_domain_cpuslocked()
1579 	struct rapl_package *rp;  in rapl_add_package_cpuslocked()  local
1582 	rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);  in rapl_add_package_cpuslocked()
1583 	if (!rp)  in rapl_add_package_cpuslocked()
1587 		rp->id = topology_logical_die_id(id);  in rapl_add_package_cpuslocked()
1588 		rp->lead_cpu = id;  in rapl_add_package_cpuslocked()
1590 			snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d",  in rapl_add_package_cpuslocked()
1593 			snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",  in rapl_add_package_cpuslocked()
1596 		rp->id = id;  in rapl_add_package_cpuslocked()
1597 		rp->lead_cpu = -1;  in rapl_add_package_cpuslocked()
1598 		snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id);  in rapl_add_package_cpuslocked()
1601 	rp->priv = priv;  in rapl_add_package_cpuslocked()
1602 	ret = rapl_config(rp);  in rapl_add_package_cpuslocked()
1607 	if (rapl_detect_domains(rp)) {  in rapl_add_package_cpuslocked()
1611 	ret = rapl_package_register_powercap(rp);  in rapl_add_package_cpuslocked()
1613 		INIT_LIST_HEAD(&rp->plist);  in rapl_add_package_cpuslocked()
1614 		list_add(&rp->plist, &rapl_packages);  in rapl_add_package_cpuslocked()
1615 		return rp;  in rapl_add_package_cpuslocked()
1619 	kfree(rp->domains);  in rapl_add_package_cpuslocked()
1620 	kfree(rp);  in rapl_add_package_cpuslocked()
1634 	struct rapl_package *rp;  in power_limit_state_save()  local
1639 	list_for_each_entry(rp, &rapl_packages, plist) {  in power_limit_state_save()
1640 		if (!rp->power_zone)  in power_limit_state_save()
1642 		rd = power_zone_to_rapl_domain(rp->power_zone);  in power_limit_state_save()
1655 	struct rapl_package *rp;  in power_limit_state_restore()  local
1660 	list_for_each_entry(rp, &rapl_packages, plist) {  in power_limit_state_restore()
1661 		if (!rp->power_zone)  in power_limit_state_restore()
1663 		rd = power_zone_to_rapl_domain(rp->power_zone);  in power_limit_state_restore()