撒

数据建模及特征处理方式，基于差分的思想基于数据处理，挖掘基于工艺历史数据的多recipe多变量参数差异变化与spec协同变化进行综合建模分析：
for i in range(len(X)):
try:
dist = compute_hamming_distance_vector(X[i], X)
# 1. 对距离向量排序，并记录原始索引
sorted_indices = np.argsort(dist)
sorted_dists = dist[sorted_indices]
distances1.append(sorted_dists)
except Exception as e:
pass # 建议打印异常信息方便调试：print(e)
distances=np.array(distances)
distances1 = np.array(distances1)

2. 计算每行的中位数

medians = np.median(distances1, axis=1)
medians_dict = dict(zip(id_to_index.keys(), medians))

import pdb;pdb.set_trace()

3. 计算中位数的中位数

median_of_medians = np.median(medians)

4. 统计每行中有多少个点在这个中位数内

counts = np.sum(distances1 <= median_of_medians, axis=1)
for base_id in base_ids:
i = id_to_index.get(base_id)
if i is None:
continue
base_vector = X[i]
dist_vector = compute_hamming_distance_vector(base_vector, X) #compute_weighted_l1_distance_vector(base_vector, X, weight_vector)
# 1. 对距离向量排序，并记录原始索引
sorted_indices = np.argsort(dist_vector)
sorted_dists = dist_vector[sorted_indices]

# 2. 计算相邻距离的差值
dist_diff = np.diff(sorted_dists)# 3. 设置一个百分位数阈值（例如 75%），用于识别跳跃点
threshold = np.percentile(dist_diff, 75)# 4. 找出第一个显著跳跃点
jump_indices = np.where(dist_diff > threshold)[0]# 5. 动态确定分界点
if len(jump_indices) > 0:cutoff_index = jump_indices[0]  # 第一个跳跃点
else:cutoff_index = int(len(sorted_dists) * 0.5)  # 默认保留前50%# 6. 保留跳跃点之前的样本（即“正常”样本）valid_idxs = sorted_indices[:cutoff_index]
valid_idxs = valid_idxs[valid_idxs != i]
#print("有效样本索引:", valid_idxs)valid_idxs = np.where((dist_vector >0) & (dist_vector <= 200))[0]
# 获取这些索引对应的 'score#' 列的值
scores = filtered_df.iloc[valid_idxs]['score#']
corr=filtered_df.iloc[valid_idxs].corr()
# 找到 'score#' 值最高的前两个索引
excluded_idxs = scores.nlargest(1).index
#data=data.iloc[valid_idxs,1:].corr()
# 记录被排除的索引
#valid_idxs = np.setdiff1d(valid_idxs, excluded_idxs)if base_id not in local_dict:local_dict[base_id] = {}
local_dict[base_id]['nei'] =   [index_to_id[item]  for item in  valid_idxs.tolist()]
local_dict[base_id]['target'] =   [index_to_id[item]  for item in  excluded_idxs.tolist()]
#输出保留下来的样本数量
print("保留的有效样本数量:", valid_idxs)# use.extend(valid_idxs)
#print('=============')
# Step 1: base_id 和 valid_idxs 中的每个点做比较
for j in valid_idxs:delta_x = X[j] - X[i]delta_y = y[j] - y[i]abs_delta_x = np.abs(delta_x)change_indices = np.where(abs_delta_x > 0.001)[0]abs_delta_x_ = np.abs(delta_x)# 找出哪些特征变化大于0.001changed_features = [(feature_cols[k], delta_x[k]) for k in change_indices]changed_features_ = [feature_cols[k] for k in change_indices]# 计算两行之间的特征差值（delta_x）delta_x_raw = data[feature_cols].iloc[j] - data[feature_cols].iloc[i]# 计算绝对值abs_delta_x_raw = delta_x_raw.abs()# 筛选变化大于 0.001 的特征（返回布尔 Series）mask = abs_delta_x_raw > 0.001# 提取变化的特征及其差值（类似 changed_features）changed_features_raw = delta_x_raw[mask].to_dict()  # 字典形式：{特征名: 差值}# 或列表形式：[(特征名, 差值), ...]changed_features_list = list(changed_features_raw.items())param_set.update(changed_features_)# 构建记录字典record = {f"Δ{col}": delta_x[k] for k, col in enumerate(feature_cols)}record[f"Δ{target_col}"] = delta_yrecord["base_id"] = ids[i]record["compare_id1"] = ids[i]record["compare_id2"] = ids[j]record["distance"] = dist_vector[j]record["change_num"] = len(changed_features)if record["change_num"] > 200:continue  # 跳过 change_num > 10 的记录# 存储变化的特征名和对应的变化值record["param_dt"] = ["{}:{:.2f}".format(feat, delta) for feat, delta in changed_features]record["param_dt_raw"] = ["{}:{:.2f}".format(feat, delta) for feat, delta in changed_features_list]record["param"] = changed_features_diffs.append(record)#Step 2: valid_idxs 内部两两之间也做比较
n = len(valid_idxs)
for idx1 in range(n):for idx2 in range(idx1 + 1, n):i1 = valid_idxs[idx1]i2 = valid_idxs[idx2]delta_x = X[i2] - X[i1]delta_y = y[i2] - y[i1]# 计算 distancedistance = np.sum(X[i1] != X[i2])  #np.sum(np.abs(X[i1] - X[i2]))# 找出变化超过 0.001 的特征列abs_delta_x = np.abs(delta_x)change_indices = np.where(abs_delta_x > 0.001)[0]change_num = len(change_indices)if change_num >2:continue  # 跳过 distance > 10 的记录changed_features = [(feature_cols[k], delta_x[k]) for k in change_indices]changed_features_ = [feature_cols[k] for k in change_indices]# 计算两行之间的特征差值（delta_x）delta_x_raw = data[feature_cols].iloc[i2] - data[feature_cols].iloc[i1]# 计算绝对值abs_delta_x_raw = delta_x_raw.abs()# 筛选变化大于 0.001 的特征（返回布尔 Series）mask = abs_delta_x_raw > 0.001# 提取变化的特征及其差值（类似 changed_features）changed_features_raw = delta_x_raw[mask].to_dict()  # 字典形式：{特征名: 差值}# 或列表形式：[(特征名, 差值), ...]changed_features_list = list(changed_features_raw.items())record = {f"Δ{col}": delta_x[k] for k, col in enumerate(feature_cols)}record[f"Δ{target_col}"] = delta_yrecord["compare_id1"] = ids[i1]record["base_id"] = ids[i]  # 假设 base_id 是 i1 的 idrecord["compare_id2"] = ids[i2]record["distance"] = distancerecord["change_num"] = change_numrecord["param_dt"] = ["{}:{:.2f}".format(feat, delta) for feat, delta in changed_features]record["param_dt_raw"] = ["{}:{:.2f}".format(feat, delta) for feat, delta in changed_features_list]record["param"] = changed_features_diffs.append(record)

use.extend(indices_with_none)

print(len(set(use)))

epsilon = 0.0001

df_diff = pd.DataFrame(diffs)

def smooth_weight(dist, scale=1.0):
"""
平滑函数：根据距离计算权重，scale 控制衰减速率
可选：指数衰减、倒数、高斯核等
"""
return 1 / (1 + dist / scale)
def remove_outliers_zscore_grouped(df, category_col, value_col, threshold=2):
# 对每个分组计算 z-score，并保留非异常值
df['z_score'] = df.groupby(category_col)[value_col].transform(lambda x: zscore(x, nan_policy='omit'))

# 筛选 z-score 绝对值小于阈值的数据（默认阈值为 3）
cleaned_df = df[np.abs(df['z_score']) < threshold]# 删除辅助列
cleaned_df = cleaned_df.drop(columns=['z_score'])return cleaned_df

from matplotlib.patches import FancyArrowPatch
import matplotlib.cm as cm
import matplotlib.colors as mcolors
def generate_distinct_colors(n):
hues = np.linspace(0, 1, n, endpoint=False)
hsv_colors = np.column_stack([hues, np.ones(n) * 0.7, np.ones(n) * 0.9])
rgb_colors = np.array([mcolors.hsv_to_rgb(c) for c in hsv_colors])
return [mcolors.rgb2hex(rgb) for rgb in rgb_colors]

def build_trie_with_frequency(paths_with_pairs):
root = {}
freq_counter = Counter()

for path, pair in paths_with_pairs:node = rootfor param in path:if param not in node:node[param] = {}node = node[param]freq_counter[param] += 1return root, freq_counter