// codesnip-guard: StaticSearch
#[cfg_attr(any(),rustfmt::skip)]pub use self::static_search::{SimdKey,StaticSearch};#[cfg_attr(any(),rustfmt::skip)]mod static_search{use super::SimdBackend;#[cfg(target_arch="x86_64")]use super::{avx512_enabled,simd};use std::{marker::PhantomData,ops::Range};#[inline]fn static_search_backend(bits:u32)->SimdBackend{#[cfg(target_arch="x86_64")]{if avx512_enabled()&&is_x86_feature_detected!("avx512f")&&(bits!=16||is_x86_feature_detected!("avx512bw")){return SimdBackend::Avx512;}if is_x86_feature_detected!("avx2"){return SimdBackend::Avx2;}}let _=bits;SimdBackend::Scalar}#[doc=" Maps a key to an unsigned integer with exactly the same ordering."]#[doc=""]#[doc=" `BITS` must be 8, 16, 32, 64, or 128. `encode` must be deterministic, fit in"]#[doc=" `BITS`, and satisfy `a.cmp(&b) == a.encode().cmp(&b.encode())`."]pub trait SimdKey:Copy+Ord{const BITS:u32;fn encode(self)->u128;}macro_rules!impl_unsigned_simd_key{($($value:ty),*$(,)?)=>{$(impl SimdKey for$value{const BITS:u32=<$value>::BITS;#[inline(always)]fn encode(self)->u128{self as u128}})*};}macro_rules!impl_signed_simd_key{($(($signed:ty,$unsigned:ty)),*$(,)?)=>{$(impl SimdKey for$signed{const BITS:u32=<$signed>::BITS;#[inline(always)]fn encode(self)->u128{((self as$unsigned)^((1 as$unsigned)<<(<$signed>::BITS-1)))as u128}})*};}impl_unsigned_simd_key!(u8,u16,u32,u64,u128,usize);impl_signed_simd_key!((i8,u8),(i16,u16),(i32,u32),(i64,u64),(i128,u128),(isize,usize),);#[derive(Clone,Debug)]enum DirectStaticSearch{U16(Vec<u16>),U32(Vec<u32>)}impl DirectStaticSearch{fn build<K:SimdKey>(values:&[K],bits:u32)->Self{assert!(values.len()<=u32::MAX as usize);if values.len()<=u16::MAX as usize{Self::U16(build_direct_positions(values,bits,|position|{position as u16}))}else{Self::U32(build_direct_positions(values,bits,|position|{position as u32}))}}#[inline(always)]fn lower_bound(&self,value:u128)->usize{let value=usize::try_from(value).expect("SimdKey::encode exceeds usize");match self{Self::U16(positions)=>positions[value]as usize,Self::U32(positions)=>positions[value]as usize,}}#[inline(always)]fn upper_bound(&self,value:u128)->usize{let value=usize::try_from(value).ok().and_then(|value|value.checked_add(1)).expect("SimdKey::encode exceeds usize");match self{Self::U16(positions)=>positions[value]as usize,Self::U32(positions)=>positions[value]as usize,}}#[inline(always)]fn contains(&self,value:u128)->bool{let value=usize::try_from(value).ok().and_then(|value|value.checked_add(1).map(|next|(value,next))).expect("SimdKey::encode exceeds usize");match self{Self::U16(positions)=>positions[value.0]!=positions[value.1],Self::U32(positions)=>positions[value.0]!=positions[value.1],}}}fn build_direct_positions<K,P>(values:&[K],bits:u32,position:impl Fn(usize)->P)->Vec<P>where K:SimdKey,P:Copy{let len=(1<<bits)+1;let mut positions=Vec::with_capacity(len);let maximum=(1u128<<bits)-1;let mut previous:Option<(K,u128)>=None;for(index,&value)in values.iter().enumerate(){let encoded=value.encode();assert!(encoded<=maximum,"SimdKey::encode exceeds its declared width");if let Some((previous_value,previous_encoded))=previous{assert_eq!(previous_value.cmp(&value),previous_encoded.cmp(&encoded),"SimdKey::encode does not preserve order");}if previous.is_none_or(|(_,previous)|previous!=encoded){positions.resize(encoded as usize+1,position(index));}previous=Some((value,encoded));}positions.resize(len,position(values.len()));positions}macro_rules!bound_batch{(lower,$this:ident,$values:ident,$search:expr)=>{{if$this.len==0{return[0;16];}let mut$values=*$values;let mut beyond=[false;16];for index in 0..16{beyond[index]=$values[index]>$this.maximum;$values[index]=$values[index].min($this.maximum);}let mut result=$search;for index in 0..16{if beyond[index]{result[index]=$this.len;}}result}};(upper,$this:ident,$values:ident,$search:expr)=>{{if$this.len==0{return[0;16];}let mut$values=*$values;let mut beyond=[false;16];for index in 0..16{beyond[index]=$values[index]>=$this.maximum;}let Some(value)=$values.iter().copied().find(|&value|value<$this.maximum)else{return[$this.len;16];};for index in 0..16{if beyond[index]{$values[index]=value;}}let mut result=$search;for index in 0..16{if beyond[index]{result[index]=$this.len;}}result}};}#[repr(C,align(64))]#[derive(Clone,Debug)]struct SearchBlock<T,const B:usize>([T;B]);#[derive(Clone,Debug)]struct StaticSearchTree<T,const B:usize>{values:Vec<SearchBlock<T,B>>,len:usize,maximum:T,levels:Vec<Vec<SearchBlock<T,B>>>,#[cfg(target_arch="x86_64")]backend:SimdBackend}impl<T:Copy+Ord,const B:usize>StaticSearchTree<T,B>{fn build<K>(values:&[K],sentinel:T,maximum_encoded:u128,convert:impl Fn(u128)->T,backend:SimdBackend)->Self where K:SimdKey{let _=&backend;let len=values.len();let mut previous:Option<(K,T)>=None;let mut separators=Vec::with_capacity(values.len().div_ceil(B));let mut blocks=Vec::with_capacity(separators.capacity());for chunk in values.chunks(B){let mut block=[sentinel;B];for(index,&value)in chunk.iter().enumerate(){let encoded=value.encode();assert!(encoded<=maximum_encoded,"SimdKey::encode exceeds its declared width");let encoded=convert(encoded);if let Some((previous_value,previous_encoded))=previous{assert!(previous_value.cmp(&value)==previous_encoded.cmp(&encoded),"SimdKey::encode does not preserve order");}previous=Some((value,encoded));block[index]=encoded;}separators.push(block[chunk.len()-1]);blocks.push(SearchBlock(block));}let maximum=separators.last().copied().unwrap_or(sentinel);let mut levels=Vec::new();while separators.len()>1{let mut blocks=Vec::with_capacity(separators.len().div_ceil(B));let mut next=Vec::with_capacity(blocks.capacity());for chunk in separators.chunks(B){let mut block=[sentinel;B];block[..chunk.len()].copy_from_slice(chunk);blocks.push(SearchBlock(block));next.push(chunk[chunk.len()-1]);}levels.push(blocks);separators=next;}Self{values:blocks,len,maximum,levels,#[cfg(target_arch="x86_64")]backend}}#[inline(always)]fn descend<F>(&self,value:T,mut position:F)->usize where F:FnMut(&[T;B],T)->usize{let mut block=0;for level in self.levels.iter().rev(){let values=&unsafe{level.get_unchecked(block)}.0;block=block*B+position(values,value);}let values=&unsafe{self.values.get_unchecked(block)}.0;(block*B+position(values,value)).min(self.len)}#[inline(always)]fn get(&self,index:usize)->T{unsafe{*self.values.get_unchecked(index/B).0 .get_unchecked(index%B)}}#[inline(always)]fn descend_batch<F>(&self,values:&[T;16],mut position:F)->[usize;16]where F:FnMut(&[T;B],T)->usize{let mut blocks=[0;16];for level in self.levels.iter().rev(){for index in 0..16{let block_values=&unsafe{level.get_unchecked(blocks[index])}.0;blocks[index]=blocks[index]*B+position(block_values,values[index]);}}for index in 0..16{let block=blocks[index];let block_values=&unsafe{self.values.get_unchecked(block)}.0;blocks[index]=(block*B+position(block_values,values[index])).min(self.len);}blocks}#[inline(always)]fn lower_bound_scalar(&self,value:T)->usize{self.descend(value,|values,value|{values.partition_point(|&current|current<value)})}#[inline(always)]fn upper_bound_scalar(&self,value:T)->usize{self.descend(value,|values,value|{values.partition_point(|&current|current<=value)})}#[inline(always)]fn lower_bound_batch_scalar(&self,values:&[T;16])->[usize;16]{self.descend_batch(values,|values,value|{values.partition_point(|&current|current<value)})}#[inline(always)]fn upper_bound_batch_scalar(&self,values:&[T;16])->[usize;16]{self.descend_batch(values,|values,value|{values.partition_point(|&current|current<=value)})}}macro_rules!impl_static_search_tree{($value:ty,$branch:expr,$first_ge_avx2:ident,$first_gt_avx2:ident,$first_ge_avx512:ident,$first_gt_avx512:ident,$avx512_features:literal)=>{impl StaticSearchTree<$value,$branch>{#[inline]fn lower_bound(&self,value:$value)->usize{if self.len==0||value>self.maximum{return self.len;}#[cfg(target_arch="x86_64")]return match self.backend{SimdBackend::Scalar=>self.lower_bound_scalar(value),SimdBackend::Avx2=>unsafe{self.lower_bound_avx2(value)},SimdBackend::Avx512=>unsafe{self.lower_bound_avx512(value)},};#[cfg(not(target_arch="x86_64"))]self.lower_bound_scalar(value)}#[inline]fn upper_bound(&self,value:$value)->usize{if self.len==0{return 0;}if value>=self.maximum{return self.len;}#[cfg(target_arch="x86_64")]return match self.backend{SimdBackend::Scalar=>self.upper_bound_scalar(value),SimdBackend::Avx2=>unsafe{self.upper_bound_avx2(value)},SimdBackend::Avx512=>unsafe{self.upper_bound_avx512(value)},};#[cfg(not(target_arch="x86_64"))]self.upper_bound_scalar(value)}#[inline]fn contains(&self,value:$value)->bool{let index=self.lower_bound(value);index<self.len&&self.get(index)==value}#[inline]fn lower_bound_batch(&self,values:&[$value;16])->[usize;16]{#[cfg(target_arch="x86_64")]if self.backend==SimdBackend::Avx512{return unsafe{self.lower_bound_batch_avx512(values)};}bound_batch!(lower,self,values,{#[cfg(target_arch="x86_64")]let result=if self.backend==SimdBackend::Avx2{unsafe{self.lower_bound_batch_avx2(&values)}}else{self.lower_bound_batch_scalar(&values)};#[cfg(not(target_arch="x86_64"))]let result=self.lower_bound_batch_scalar(&values);result})}#[inline]fn upper_bound_batch(&self,values:&[$value;16])->[usize;16]{#[cfg(target_arch="x86_64")]if self.backend==SimdBackend::Avx512{return unsafe{self.upper_bound_batch_avx512(values)};}bound_batch!(upper,self,values,{#[cfg(target_arch="x86_64")]let result=if self.backend==SimdBackend::Avx2{unsafe{self.upper_bound_batch_avx2(&values)}}else{self.upper_bound_batch_scalar(&values)};#[cfg(not(target_arch="x86_64"))]let result=self.upper_bound_batch_scalar(&values);result})}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]unsafe fn lower_bound_avx2(&self,value:$value)->usize{self.descend(value,|values,value|unsafe{simd::$first_ge_avx2(values,value)})}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]unsafe fn upper_bound_avx2(&self,value:$value)->usize{self.descend(value,|values,value|unsafe{simd::$first_gt_avx2(values,value)})}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]unsafe fn lower_bound_batch_avx2(&self,values:&[$value;16])->[usize;16]{self.descend_batch(values,|values,value|unsafe{simd::$first_ge_avx2(values,value)})}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]unsafe fn upper_bound_batch_avx2(&self,values:&[$value;16])->[usize;16]{self.descend_batch(values,|values,value|unsafe{simd::$first_gt_avx2(values,value)})}#[cfg(target_arch="x86_64")]#[target_feature(enable=$avx512_features)]unsafe fn lower_bound_avx512(&self,value:$value)->usize{self.descend(value,|values,value|unsafe{simd::$first_ge_avx512(values,value)})}#[cfg(target_arch="x86_64")]#[target_feature(enable=$avx512_features)]unsafe fn upper_bound_avx512(&self,value:$value)->usize{self.descend(value,|values,value|unsafe{simd::$first_gt_avx512(values,value)})}#[cfg(target_arch="x86_64")]#[target_feature(enable=$avx512_features)]unsafe fn lower_bound_batch_avx512(&self,values:&[$value;16])->[usize;16]{bound_batch!(lower,self,values,self.descend_batch(&values,|values,value|unsafe{simd::$first_ge_avx512(values,value)}))}#[cfg(target_arch="x86_64")]#[target_feature(enable=$avx512_features)]unsafe fn upper_bound_batch_avx512(&self,values:&[$value;16])->[usize;16]{bound_batch!(upper,self,values,self.descend_batch(&values,|values,value|unsafe{simd::$first_gt_avx512(values,value)}))}}};}impl_static_search_tree!(u16,32,first_ge_u16x32_avx2,first_gt_u16x32_avx2,first_ge_u16x32_avx512,first_gt_u16x32_avx512,"avx512f,avx512bw");impl_static_search_tree!(u32,16,first_ge_u32x16_avx2,first_gt_u32x16_avx2,first_ge_u32x16_avx512,first_gt_u32x16_avx512,"avx512f");impl_static_search_tree!(u64,8,first_ge_u64x8_avx2,first_gt_u64x8_avx2,first_ge_u64x8_avx512,first_gt_u64x8_avx512,"avx512f");impl StaticSearchTree<u128,4>{#[inline]fn lower_bound(&self,value:u128)->usize{if self.len==0||value>self.maximum{self.len}else{self.descend(value,|values,value|{(values[0]<value)as usize+(values[1]<value)as usize+(values[2]<value)as usize+(values[3]<value)as usize})}}#[inline]fn upper_bound(&self,value:u128)->usize{if self.len==0||value>=self.maximum{self.len}else{self.descend(value,|values,value|{(values[0]<=value)as usize+(values[1]<=value)as usize+(values[2]<=value)as usize+(values[3]<=value)as usize})}}#[inline]fn contains(&self,value:u128)->bool{let index=self.lower_bound(value);index<self.len&&self.get(index)==value}#[inline]fn lower_bound_batch(&self,values:&[u128;16])->[usize;16]{bound_batch!(lower,self,values,self.descend_batch(&values,|values,value|{(values[0]<value)as usize+(values[1]<value)as usize+(values[2]<value)as usize+(values[3]<value)as usize}))}#[inline]fn upper_bound_batch(&self,values:&[u128;16])->[usize;16]{bound_batch!(upper,self,values,self.descend_batch(&values,|values,value|{(values[0]<=value)as usize+(values[1]<=value)as usize+(values[2]<=value)as usize+(values[3]<=value)as usize}))}}fn search_batch<K,T>(values:&[K],output:&mut[usize],convert:impl Fn(u128)->T,single:impl Fn(T)->usize,batch:impl Fn(&[T;16])->[usize;16])where K:SimdKey,T:Copy{let mut offset=0;while offset+16<=values.len(){let values=std::array::from_fn(|index|convert(values[offset+index].encode()));output[offset..offset+16].copy_from_slice(&batch(&values));offset+=16;}let remaining=values.len()-offset;if remaining>=8{let mut encoded=[convert(values[offset].encode());16];for index in 1..remaining{encoded[index]=convert(values[offset+index].encode());}let positions=batch(&encoded);output[offset..].copy_from_slice(&positions[..remaining]);}else{for(&value,position)in values[offset..].iter().zip(&mut output[offset..]){*position=single(convert(value.encode()));}}}#[derive(Clone,Debug)]enum StaticSearchStorage{Direct(DirectStaticSearch),U16(StaticSearchTree<u16,32>),U32(StaticSearchTree<u32,16>),U64(StaticSearchTree<u64,8>),U128(StaticSearchTree<u128,4>)}impl StaticSearchStorage{#[inline(always)]fn lower_bound(&self,value:u128)->usize{match self{Self::Direct(search)=>search.lower_bound(value),Self::U16(search)=>search.lower_bound(u16::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U32(search)=>search.lower_bound(u32::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U64(search)=>search.lower_bound(u64::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U128(search)=>search.lower_bound(value),}}#[inline(always)]fn upper_bound(&self,value:u128)->usize{match self{Self::Direct(search)=>search.upper_bound(value),Self::U16(search)=>search.upper_bound(u16::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U32(search)=>search.upper_bound(u32::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U64(search)=>search.upper_bound(u64::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U128(search)=>search.upper_bound(value),}}#[inline(always)]fn contains(&self,value:u128)->bool{match self{Self::Direct(search)=>search.contains(value),Self::U16(search)=>search.contains(u16::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U32(search)=>search.contains(u32::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U64(search)=>search.contains(u64::try_from(value).expect("SimdKey::encode exceeds its declared width")),Self::U128(search)=>search.contains(value),}}fn lower_bound_batch<K:SimdKey>(&self,values:&[K],output:&mut[usize]){match self{Self::Direct(search)=>{for(&value,position)in values.iter().zip(output){*position=search.lower_bound(value.encode());}}Self::U16(search)=>search_batch(values,output,|value|u16::try_from(value).expect("SimdKey::encode exceeds its declared width"),|value|search.lower_bound(value),|values|search.lower_bound_batch(values)),Self::U32(search)=>search_batch(values,output,|value|u32::try_from(value).expect("SimdKey::encode exceeds its declared width"),|value|search.lower_bound(value),|values|search.lower_bound_batch(values)),Self::U64(search)=>search_batch(values,output,|value|u64::try_from(value).expect("SimdKey::encode exceeds its declared width"),|value|search.lower_bound(value),|values|search.lower_bound_batch(values)),Self::U128(search)=>search_batch(values,output,|value|value,|value|search.lower_bound(value),|values|search.lower_bound_batch(values)),}}fn upper_bound_batch<K:SimdKey>(&self,values:&[K],output:&mut[usize]){match self{Self::Direct(search)=>{for(&value,position)in values.iter().zip(output){*position=search.upper_bound(value.encode());}}Self::U16(search)=>search_batch(values,output,|value|u16::try_from(value).expect("SimdKey::encode exceeds its declared width"),|value|search.upper_bound(value),|values|search.upper_bound_batch(values)),Self::U32(search)=>search_batch(values,output,|value|u32::try_from(value).expect("SimdKey::encode exceeds its declared width"),|value|search.upper_bound(value),|values|search.upper_bound_batch(values)),Self::U64(search)=>search_batch(values,output,|value|u64::try_from(value).expect("SimdKey::encode exceeds its declared width"),|value|search.upper_bound(value),|values|search.upper_bound_batch(values)),Self::U128(search)=>search_batch(values,output,|value|value,|value|search.upper_bound(value),|values|search.upper_bound_batch(values)),}}}#[doc=" A static search index over sorted integer or integer-encoded keys."]#[doc=""]#[doc=" The index adds build time and storage. For a small number of searches, search the sorted slice"]#[doc=" directly instead."]#[derive(Clone,Debug)]pub struct StaticSearch<K>{storage:StaticSearchStorage,len:usize,marker:PhantomData<fn()->K>}impl<K:SimdKey>StaticSearch<K>{#[doc=" Builds an index over sorted `values`."]#[doc=""]#[doc=" # Panics"]#[doc=""]#[doc=" Panics if `values` is not sorted or `SimdKey` does not satisfy its contract."]pub fn from_sorted(values:&[K])->Self{Self::build(values,static_search_backend(K::BITS),false)}#[doc=" Builds a direct lookup table over sorted 8-bit or 16-bit `values`."]#[doc=""]#[doc=" This layout uses a fixed table of 257 or 65,537 positions and is intended"]#[doc=" for query-heavy workloads."]#[doc=""]#[doc=" # Panics"]#[doc=""]#[doc=" Panics if `K::BITS` is neither 8 nor 16, or if `values` is not sorted."]pub fn from_sorted_direct(values:&[K])->Self{assert!(matches!(K::BITS,8|16));Self::build(values,static_search_backend(K::BITS),true)}#[inline]pub fn len(&self)->usize{self.len}#[inline]pub fn is_empty(&self)->bool{self.len==0}#[doc=" Returns the first index whose value is greater than or equal to `value`."]#[inline]pub fn lower_bound(&self,value:K)->usize{self.storage.lower_bound(value.encode())}#[doc=" Returns one past the last index whose value is less than or equal to `value`."]#[inline]pub fn upper_bound(&self,value:K)->usize{self.storage.upper_bound(value.encode())}#[doc=" Writes the first index greater than or equal to each value into `output`."]#[doc=""]#[doc=" # Panics"]#[doc=""]#[doc=" Panics if `values` and `output` have different lengths."]pub fn lower_bound_batch(&self,values:&[K],output:&mut[usize]){assert_eq!(values.len(),output.len());self.storage.lower_bound_batch(values,output);}#[doc=" Writes one past the last index less than or equal to each value into `output`."]#[doc=""]#[doc=" # Panics"]#[doc=""]#[doc=" Panics if `values` and `output` have different lengths."]pub fn upper_bound_batch(&self,values:&[K],output:&mut[usize]){assert_eq!(values.len(),output.len());self.storage.upper_bound_batch(values,output);}#[inline]pub fn range(&self,value:K)->Range<usize>{let value=value.encode();self.storage.lower_bound(value)..self.storage.upper_bound(value)}#[inline]pub fn contains(&self,value:K)->bool{self.storage.contains(value.encode())}fn build(values:&[K],backend:SimdBackend,direct:bool)->Self{assert!(matches!(K::BITS,8|16|32|64|128));assert!(values.windows(2).all(|pair|pair[0]<=pair[1]));let len=values.len();let storage=match K::BITS{8=>StaticSearchStorage::Direct(DirectStaticSearch::build(values,K::BITS)),16=>{if direct{StaticSearchStorage::Direct(DirectStaticSearch::build(values,K::BITS))}else{StaticSearchStorage::U16(StaticSearchTree::build(values,u16::MAX,u16::MAX as u128,|value|value as u16,backend))}}32=>StaticSearchStorage::U32(StaticSearchTree::build(values,u32::MAX,u32::MAX as u128,|value|value as u32,backend)),64=>StaticSearchStorage::U64(StaticSearchTree::build(values,u64::MAX,u64::MAX as u128,|value|value as u64,backend)),128=>StaticSearchStorage::U128(StaticSearchTree::build(values,u128::MAX,u128::MAX,|value|value,backend)),_=>unreachable!(),};Self{storage,len,marker:PhantomData}}}}
// codesnip-guard: _simd
#[cfg_attr(any(),rustfmt::skip)]pub mod simd{#![allow(unsafe_op_in_unsafe_fn)]#[cfg(target_arch="x86_64")]use std::arch::x86_64::*;#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_ge_u16x32_avx2(values:&[u16;32],key:u16)->usize{let sign=_mm256_set1_epi16(i16::MIN);let key=_mm256_xor_si256(_mm256_set1_epi16(key as i16),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(16).cast()),sign);let low=_mm256_movemask_epi8(_mm256_cmpgt_epi16(key,low))as u32 as u64;let high=_mm256_movemask_epi8(_mm256_cmpgt_epi16(key,high))as u32 as u64;(!(low|high<<32)).trailing_zeros()as usize/2}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_gt_u16x32_avx2(values:&[u16;32],key:u16)->usize{let sign=_mm256_set1_epi16(i16::MIN);let key=_mm256_xor_si256(_mm256_set1_epi16(key as i16),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(16).cast()),sign);let low=_mm256_movemask_epi8(_mm256_cmpgt_epi16(low,key))as u32 as u64;let high=_mm256_movemask_epi8(_mm256_cmpgt_epi16(high,key))as u32 as u64;(low|high<<32).trailing_zeros()as usize/2}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_ge_u32x16_avx2(values:&[u32;16],key:u32)->usize{let sign=_mm256_set1_epi32(i32::MIN);let key=_mm256_xor_si256(_mm256_set1_epi32(key as i32),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(8).cast()),sign);let low=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(key,low)));let high=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(key,high)));let mask=((!low as u32)&0xff)|(((!high as u32)&0xff)<<8);(mask as u16).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_gt_u32x16_avx2(values:&[u32;16],key:u32)->usize{let sign=_mm256_set1_epi32(i32::MIN);let key=_mm256_xor_si256(_mm256_set1_epi32(key as i32),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(8).cast()),sign);let low=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(low,key)));let high=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(high,key)));let mask=low as u32|((high as u32)<<8);(mask as u16).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_ge_u64x8_avx2(values:&[u64;8],key:u64)->usize{let sign=_mm256_set1_epi64x(i64::MIN);let key=_mm256_xor_si256(_mm256_set1_epi64x(key as i64),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),sign);let low=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(key,low)));let high=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(key,high)));let mask=((!low as u32)&0x0f)|(((!high as u32)&0x0f)<<4);(mask as u8).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn first_gt_u64x8_avx2(values:&[u64;8],key:u64)->usize{let sign=_mm256_set1_epi64x(i64::MIN);let key=_mm256_xor_si256(_mm256_set1_epi64x(key as i64),sign);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),sign);let low=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(low,key)));let high=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(high,key)));let mask=low as u32|((high as u32)<<4);(mask as u8).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f,avx512bw")]#[inline]pub unsafe fn first_ge_u16x32_avx512(values:&[u16;32],key:u16)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi16(key as i16);(!_mm512_cmplt_epu16_mask(values,key)).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f,avx512bw")]#[inline]pub unsafe fn first_gt_u16x32_avx512(values:&[u16;32],key:u16)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi16(key as i16);_mm512_cmpgt_epu16_mask(values,key).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_ge_u32x16_avx512(values:&[u32;16],key:u32)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi32(key as i32);(!_mm512_cmplt_epu32_mask(values,key)).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_gt_u32x16_avx512(values:&[u32;16],key:u32)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi32(key as i32);_mm512_cmpgt_epu32_mask(values,key).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_ge_u64x8_avx512(values:&[u64;8],key:u64)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi64(key as i64);(!_mm512_cmplt_epu64_mask(values,key)).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn first_gt_u64x8_avx512(values:&[u64;8],key:u64)->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let key=_mm512_set1_epi64(key as i64);_mm512_cmpgt_epu64_mask(values,key).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn add_suffix_u32x16_avx2(values:&mut[u32;16],index:usize,delta:u32){let index=_mm256_set1_epi32(index as i32-1);let delta=_mm256_set1_epi32(delta as i32);let low_mask=_mm256_cmpgt_epi32(_mm256_setr_epi32(0,1,2,3,4,5,6,7),index);let high_mask=_mm256_cmpgt_epi32(_mm256_setr_epi32(8,9,10,11,12,13,14,15),index);let low=_mm256_add_epi32(_mm256_loadu_si256(values.as_ptr().cast()),_mm256_and_si256(delta,low_mask));let high=_mm256_add_epi32(_mm256_loadu_si256(values.as_ptr().add(8).cast()),_mm256_and_si256(delta,high_mask));_mm256_storeu_si256(values.as_mut_ptr().cast(),low);_mm256_storeu_si256(values.as_mut_ptr().add(8).cast(),high);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn add_suffix_u64x8_avx2(values:&mut[u64;8],index:usize,delta:u64){let index=_mm256_set1_epi64x(index as i64-1);let delta=_mm256_set1_epi64x(delta as i64);let low_mask=_mm256_cmpgt_epi64(_mm256_setr_epi64x(0,1,2,3),index);let high_mask=_mm256_cmpgt_epi64(_mm256_setr_epi64x(4,5,6,7),index);let low=_mm256_add_epi64(_mm256_loadu_si256(values.as_ptr().cast()),_mm256_and_si256(delta,low_mask));let high=_mm256_add_epi64(_mm256_loadu_si256(values.as_ptr().add(4).cast()),_mm256_and_si256(delta,high_mask));_mm256_storeu_si256(values.as_mut_ptr().cast(),low);_mm256_storeu_si256(values.as_mut_ptr().add(4).cast(),high);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn add_suffix_u32x16_avx512(values:&mut[u32;16],index:usize,delta:u32){let values_vector=_mm512_loadu_si512(values.as_ptr().cast());let values_vector=_mm512_mask_add_epi32(values_vector,u16::MAX<<index,values_vector,_mm512_set1_epi32(delta as i32));_mm512_storeu_si512(values.as_mut_ptr().cast(),values_vector);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn add_suffix_u64x8_avx512(values:&mut[u64;8],index:usize,delta:u64){let values_vector=_mm512_loadu_si512(values.as_ptr().cast());let values_vector=_mm512_mask_add_epi64(values_vector,u8::MAX<<index,values_vector,_mm512_set1_epi64(delta as i64));_mm512_storeu_si512(values.as_mut_ptr().cast(),values_vector);}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn max_index_u32x16_avx2(values:&[u32;16])->usize{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());let mut maximum=_mm256_max_epu32(low,high);maximum=_mm256_max_epu32(maximum,_mm256_permute2x128_si256::<0x01>(maximum,maximum));maximum=_mm256_max_epu32(maximum,_mm256_shuffle_epi32::<0x4e>(maximum));maximum=_mm256_max_epu32(maximum,_mm256_shuffle_epi32::<0xb1>(maximum));let maximum=_mm256_set1_epi32(_mm256_extract_epi32::<0>(maximum));let low=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpeq_epi32(low,maximum)));let high=_mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpeq_epi32(high,maximum)));((low as u32|((high as u32)<<8))as u16).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn max_index_u32x16_avx512(values:&[u32;16])->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let maximum=_mm512_set1_epi32(_mm512_reduce_max_epu32(values)as i32);_mm512_cmpeq_epi32_mask(values,maximum).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn max_epu64(left:__m256i,right:__m256i)->__m256i{let sign=_mm256_set1_epi64x(i64::MIN);let greater=_mm256_cmpgt_epi64(_mm256_xor_si256(left,sign),_mm256_xor_si256(right,sign));_mm256_blendv_epi8(right,left,greater)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn max_index_u64x8_avx2(values:&[u64;8])->usize{let sign=_mm256_set1_epi64x(i64::MIN);let low=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().cast()),sign);let high=_mm256_xor_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),sign);let mut maximum=max_i64x4(low,high);maximum=max_i64x4(maximum,_mm256_permute4x64_epi64::<0x4e>(maximum));maximum=max_i64x4(maximum,_mm256_permute4x64_epi64::<0xb1>(maximum));let low=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(low,maximum)));let high=_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(high,maximum)));((low as u32|((high as u32)<<4))as u8).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn max_index_u64x8_avx512(values:&[u64;8])->usize{let values=_mm512_loadu_si512(values.as_ptr().cast());let maximum=_mm512_set1_epi64(_mm512_reduce_max_epu64(values)as i64);_mm512_cmpeq_epi64_mask(values,maximum).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn max_index_u128x4_avx2(low:&[u64;4],high:&[u64;4])->usize{let low=_mm256_loadu_si256(low.as_ptr().cast());let high=_mm256_loadu_si256(high.as_ptr().cast());let mut maximum_high=high;maximum_high=max_epu64(maximum_high,_mm256_permute4x64_epi64::<0x4e>(maximum_high));maximum_high=max_epu64(maximum_high,_mm256_permute4x64_epi64::<0xb1>(maximum_high));let high_equal=_mm256_cmpeq_epi64(high,maximum_high);let high_mask=_mm256_movemask_pd(_mm256_castsi256_pd(high_equal))as u32&15;if high_mask.is_power_of_two(){return high_mask.trailing_zeros()as usize;}let mut maximum_low=_mm256_and_si256(low,high_equal);maximum_low=max_epu64(maximum_low,_mm256_permute4x64_epi64::<0x4e>(maximum_low));maximum_low=max_epu64(maximum_low,_mm256_permute4x64_epi64::<0xb1>(maximum_low));let both=_mm256_and_si256(high_equal,_mm256_cmpeq_epi64(low,maximum_low));(_mm256_movemask_pd(_mm256_castsi256_pd(both))as u32&15).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2,avx512f,avx512vl")]#[inline]pub unsafe fn max_index_u128x4_avx512(low:&[u64;4],high:&[u64;4])->usize{let low=_mm256_loadu_si256(low.as_ptr().cast());let high=_mm256_loadu_si256(high.as_ptr().cast());let mut maximum_high=high;maximum_high=_mm256_max_epu64(maximum_high,_mm256_permute4x64_epi64::<0x4e>(maximum_high));maximum_high=_mm256_max_epu64(maximum_high,_mm256_permute4x64_epi64::<0xb1>(maximum_high));let high_equal=_mm256_cmpeq_epi64(high,maximum_high);let high_mask=_mm256_movemask_pd(_mm256_castsi256_pd(high_equal))as u32&15;if high_mask.is_power_of_two(){return high_mask.trailing_zeros()as usize;}let mut maximum_low=_mm256_and_si256(low,high_equal);maximum_low=_mm256_max_epu64(maximum_low,_mm256_permute4x64_epi64::<0x4e>(maximum_low));maximum_low=_mm256_max_epu64(maximum_low,_mm256_permute4x64_epi64::<0xb1>(maximum_low));let both=_mm256_and_si256(high_equal,_mm256_cmpeq_epi64(low,maximum_low));(_mm256_movemask_pd(_mm256_castsi256_pd(both))as u32&15).trailing_zeros()as usize}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_min_i32x8(mut values:__m256i)->i32{values=_mm256_min_epi32(values,_mm256_permute2x128_si256::<0x01>(values,values));values=_mm256_min_epi32(values,_mm256_shuffle_epi32::<0x4e>(values));values=_mm256_min_epi32(values,_mm256_shuffle_epi32::<0xb1>(values));_mm256_extract_epi32::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_max_i32x8(mut values:__m256i)->i32{values=_mm256_max_epi32(values,_mm256_permute2x128_si256::<0x01>(values,values));values=_mm256_max_epi32(values,_mm256_shuffle_epi32::<0x4e>(values));values=_mm256_max_epi32(values,_mm256_shuffle_epi32::<0xb1>(values));_mm256_extract_epi32::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_sum_i32x8(mut values:__m256i)->i32{values=_mm256_add_epi32(values,_mm256_permute2x128_si256::<0x01>(values,values));values=_mm256_add_epi32(values,_mm256_shuffle_epi32::<0x4e>(values));values=_mm256_add_epi32(values,_mm256_shuffle_epi32::<0xb1>(values));_mm256_extract_epi32::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_i32x16_avx2(values:&[i32;16])->i32{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());reduce_min_i32x8(_mm256_min_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_i32x16_avx2(values:&[i32;16])->i32{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());reduce_max_i32x8(_mm256_max_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_i32x16_avx2(values:&[i32;16])->i32{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(8).cast());reduce_sum_i32x8(_mm256_add_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn range_mask_i32x8(start:usize,end:usize,offset:i32)->__m256i{let lanes=_mm256_setr_epi32(offset,offset+1,offset+2,offset+3,offset+4,offset+5,offset+6,offset+7);_mm256_and_si256(_mm256_cmpgt_epi32(lanes,_mm256_set1_epi32(start as i32-1)),_mm256_cmpgt_epi32(_mm256_set1_epi32(end as i32),lanes))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_range_i32x16_avx2(values:&[i32;16],start:usize,end:usize)->i32{let unit=_mm256_set1_epi32(i32::MAX);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i32x8(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(8).cast()),range_mask_i32x8(start,end,8));reduce_min_i32x8(_mm256_min_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_range_i32x16_avx2(values:&[i32;16],start:usize,end:usize)->i32{let unit=_mm256_set1_epi32(i32::MIN);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i32x8(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(8).cast()),range_mask_i32x8(start,end,8));reduce_max_i32x8(_mm256_max_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_range_i32x16_avx2(values:&[i32;16],start:usize,end:usize)->i32{let low=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i32x8(start,end,0));let high=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().add(8).cast()),range_mask_i32x8(start,end,8));reduce_sum_i32x8(_mm256_add_epi32(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_i32x16_avx512(values:&[i32;16])->i32{_mm512_reduce_min_epi32(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_i32x16_avx512(values:&[i32;16])->i32{_mm512_reduce_max_epi32(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_i32x16_avx512(values:&[i32;16])->i32{_mm512_reduce_add_epi32(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_range_i32x16_avx512(values:&[i32;16],start:usize,end:usize)->i32{let mask=(u16::MAX<<start)&(u16::MAX>>(16-end));_mm512_mask_reduce_min_epi32(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_range_i32x16_avx512(values:&[i32;16],start:usize,end:usize)->i32{let mask=(u16::MAX<<start)&(u16::MAX>>(16-end));_mm512_mask_reduce_max_epi32(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_range_i32x16_avx512(values:&[i32;16],start:usize,end:usize)->i32{let mask=(u16::MAX<<start)&(u16::MAX>>(16-end));_mm512_mask_reduce_add_epi32(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn min_i64x4(left:__m256i,right:__m256i)->__m256i{_mm256_blendv_epi8(left,right,_mm256_cmpgt_epi64(left,right))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn max_i64x4(left:__m256i,right:__m256i)->__m256i{_mm256_blendv_epi8(right,left,_mm256_cmpgt_epi64(left,right))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_min_i64x4(mut values:__m256i)->i64{values=min_i64x4(values,_mm256_permute4x64_epi64::<0x4e>(values));values=min_i64x4(values,_mm256_permute4x64_epi64::<0xb1>(values));_mm256_extract_epi64::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_max_i64x4(mut values:__m256i)->i64{values=max_i64x4(values,_mm256_permute4x64_epi64::<0x4e>(values));values=max_i64x4(values,_mm256_permute4x64_epi64::<0xb1>(values));_mm256_extract_epi64::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn reduce_sum_i64x4(mut values:__m256i)->i64{values=_mm256_add_epi64(values,_mm256_permute4x64_epi64::<0x4e>(values));values=_mm256_add_epi64(values,_mm256_permute4x64_epi64::<0xb1>(values));_mm256_extract_epi64::<0>(values)}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_i64x8_avx2(values:&[i64;8])->i64{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(4).cast());reduce_min_i64x4(min_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_i64x8_avx2(values:&[i64;8])->i64{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(4).cast());reduce_max_i64x4(max_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_i64x8_avx2(values:&[i64;8])->i64{let low=_mm256_loadu_si256(values.as_ptr().cast());let high=_mm256_loadu_si256(values.as_ptr().add(4).cast());reduce_sum_i64x4(_mm256_add_epi64(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]unsafe fn range_mask_i64x4(start:usize,end:usize,offset:i64)->__m256i{let lanes=_mm256_setr_epi64x(offset,offset+1,offset+2,offset+3);_mm256_and_si256(_mm256_cmpgt_epi64(lanes,_mm256_set1_epi64x(start as i64-1)),_mm256_cmpgt_epi64(_mm256_set1_epi64x(end as i64),lanes))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn minimum_range_i64x8_avx2(values:&[i64;8],start:usize,end:usize)->i64{let unit=_mm256_set1_epi64x(i64::MAX);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i64x4(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(4).cast()),range_mask_i64x4(start,end,4));reduce_min_i64x4(min_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn maximum_range_i64x8_avx2(values:&[i64;8],start:usize,end:usize)->i64{let unit=_mm256_set1_epi64x(i64::MIN);let low=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i64x4(start,end,0));let high=_mm256_blendv_epi8(unit,_mm256_loadu_si256(values.as_ptr().add(4).cast()),range_mask_i64x4(start,end,4));reduce_max_i64x4(max_i64x4(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx2")]#[inline]pub unsafe fn sum_range_i64x8_avx2(values:&[i64;8],start:usize,end:usize)->i64{let low=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().cast()),range_mask_i64x4(start,end,0));let high=_mm256_and_si256(_mm256_loadu_si256(values.as_ptr().add(4).cast()),range_mask_i64x4(start,end,4));reduce_sum_i64x4(_mm256_add_epi64(low,high))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_i64x8_avx512(values:&[i64;8])->i64{_mm512_reduce_min_epi64(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_i64x8_avx512(values:&[i64;8])->i64{_mm512_reduce_max_epi64(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_i64x8_avx512(values:&[i64;8])->i64{_mm512_reduce_add_epi64(_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn minimum_range_i64x8_avx512(values:&[i64;8],start:usize,end:usize)->i64{let mask=(u8::MAX<<start)&(u8::MAX>>(8-end));_mm512_mask_reduce_min_epi64(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn maximum_range_i64x8_avx512(values:&[i64;8],start:usize,end:usize)->i64{let mask=(u8::MAX<<start)&(u8::MAX>>(8-end));_mm512_mask_reduce_max_epi64(mask,_mm512_loadu_si512(values.as_ptr().cast()))}#[cfg(target_arch="x86_64")]#[target_feature(enable="avx512f")]#[inline]pub unsafe fn sum_range_i64x8_avx512(values:&[i64;8],start:usize,end:usize)->i64{let mask=(u8::MAX<<start)&(u8::MAX>>(8-end));_mm512_mask_reduce_add_epi64(mask,_mm512_loadu_si512(values.as_ptr().cast()))}}
// codesnip-guard: avx_helper
#[cfg_attr(any(),rustfmt::skip)]pub use self::avx_helper::{SimdBackend,avx512_enabled,avx512_supported,disable_avx512,enable_avx512,simd_backend};#[cfg_attr(any(),rustfmt::skip)]mod avx_helper{use std::sync::atomic::{AtomicBool,Ordering};#[derive(Copy,Clone,Debug,Eq,PartialEq)]pub enum SimdBackend{Scalar,Avx2,Avx512}static AVX512_ENABLED:AtomicBool=AtomicBool::new(true);#[inline]pub fn disable_avx512(){AVX512_ENABLED.store(false,Ordering::Relaxed);}#[inline]pub fn enable_avx512(){AVX512_ENABLED.store(true,Ordering::Relaxed);}#[inline]pub fn avx512_enabled()->bool{AVX512_ENABLED.load(Ordering::Relaxed)}#[inline]pub fn avx512_supported()->bool{#[cfg(any(target_arch="x86",target_arch="x86_64"))]return is_x86_feature_detected!("avx512f")&&is_x86_feature_detected!("avx512dq")&&is_x86_feature_detected!("avx512cd")&&is_x86_feature_detected!("avx512bw")&&is_x86_feature_detected!("avx512vl");#[cfg(not(any(target_arch="x86",target_arch="x86_64")))]false}#[inline]pub fn simd_backend()->SimdBackend{#[cfg(any(target_arch="x86",target_arch="x86_64"))]{if avx512_enabled()&&avx512_supported(){return SimdBackend::Avx512;}if is_x86_feature_detected!("avx2"){return SimdBackend::Avx2;}}SimdBackend::Scalar}#[macro_export]macro_rules!avx_helper{(@dispatch$backend:path,$kind:ident;$avx512:expr,$avx2:expr,$scalar:expr)=>{{#[cfg(target_arch="x86_64")]{match$backend(){$kind::Avx512=>$avx512,$kind::Avx2=>$avx2,$kind::Scalar=>$scalar,}}#[cfg(not(target_arch="x86_64"))]$scalar}};(@dispatch_avx2_fma$avx2:expr,$scalar:expr)=>{{#[cfg(target_arch="x86_64")]{if is_x86_feature_detected!("avx2")&&is_x86_feature_detected!("fma"){$avx2}else{$scalar}}#[cfg(not(target_arch="x86_64"))]$scalar}};(@avx512$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($i:ident:$t:ty),*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{$(#[$meta])*$vis fn$name$(<$($T)*>)?($($i:$t),*)->$ret where$($clauses)*{if$crate::avx512_supported(){$crate::avx_helper!(@def_avx512 fn avx512$(<$($T)*>)?($($i:$t),*)->$ret where[$($clauses)*]$body);unsafe{avx512$(::<$($T),*>)?($($i),*)}}else if is_x86_feature_detected!("avx2"){$crate::avx_helper!(@def_avx2 fn avx2$(<$($T)*>)?($($i:$t),*)->$ret where[$($clauses)*]$body);unsafe{avx2$(::<$($T),*>)?($($i),*)}}else{$body}}};(@avx2$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($i:ident:$t:ty),*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{$(#[$meta])*$vis fn$name$(<$($T)*>)?($($i:$t),*)->$ret where$($clauses)*{if is_x86_feature_detected!("avx2"){$crate::avx_helper!(@def_avx2 fn avx2$(<$($T)*>)?($($i:$t),*)->$ret where[$($clauses)*]$body);unsafe{avx2$(::<$($T),*>)?($($i),*)}}else{$body}}};(@def_avx512 fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{#[target_feature(enable="avx512f,avx512dq,avx512cd,avx512bw,avx512vl")]unsafe fn$name$(<$($T)*>)?($($args)*)->$ret where$($clauses)*$body};(@def_avx2 fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)->$ret:ty where[$($clauses:tt)*]$body:block)=>{#[target_feature(enable="avx2")]unsafe fn$name$(<$($T)*>)?($($args)*)->$ret where$($clauses)*$body};(@$tag:ident$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)->$ret:ty$body:block)=>{$crate::avx_helper!(@$tag$(#[$meta])*$vis fn$name$(<$($T)*>)?($($args)*)->$ret where[]$body);};(@$tag:ident$(#[$meta:meta])*$vis:vis fn$name:ident$(<$($T:ident),+>)?($($args:tt)*)$($t:tt)*)=>{$crate::avx_helper!(@$tag$(#[$meta])*$vis fn$name$(<$($T)*>)?($($args)*)->()$($t)*);};($($t:tt)*)=>{::std::compile_error!($($t)*);}}}