Difference between revisions of "VScaler: Advantech PCIe passthrough"
Jump to navigation
Jump to search
| Line 1: | Line 1: | ||
<syntaxhighlight> | <syntaxhighlight> | ||
| + | [root@localhost ~]# vi /etc/kolla/config/nova.conf | ||
[root@localhost ~]# cat /etc/kolla/config/nova.conf | [root@localhost ~]# cat /etc/kolla/config/nova.conf | ||
[filter_scheduler] | [filter_scheduler] | ||
| Line 12: | Line 13: | ||
passthrough_whitelist = { "address": "0000:05:00.0" } | passthrough_whitelist = { "address": "0000:05:00.0" } | ||
alias={"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"} | alias={"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"} | ||
| + | |||
| + | [root@localhost ~]# kolla-ansible -i all-in-one reconfigure | ||
| + | |||
| + | |||
</syntaxhighlight> | </syntaxhighlight> | ||
Revision as of 21:58, 6 June 2019
[root@localhost ~]# vi /etc/kolla/config/nova.conf
[root@localhost ~]# cat /etc/kolla/config/nova.conf
[filter_scheduler]
enabled_filters = RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
available_filters=nova.scheduler.filters.all_filters
[libvirt]
cpu_mode = custom
cpu_model = Skylake-Client
[pci]
passthrough_whitelist = { "address": "0000:05:00.0" }
alias={"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"}
[root@localhost ~]# kolla-ansible -i all-in-one reconfigure#check nova-scheduler , nova-api , nova-compute all have the setting
docker exec -it nova_compute cat /etc/nova/nova.conf
....
[filter_scheduler]
enabled_filters = RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
available_filters = nova.scheduler.filters.all_filters
[pci]
passthrough_whitelist = { "address": "0000:05:00.0" }
alias = {"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"}
docker exec -it nova_api cat /etc/nova/nova.conf
....
[filter_scheduler]
enabled_filters = RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
available_filters = nova.scheduler.filters.all_filters
[pci]
passthrough_whitelist = { "address": "0000:05:00.0" }
alias = {"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"}
docker exec -it nova_compute cat /etc/nova/nova.confdocker
[root@localhost ~]# openstack image create --disk-format raw --container-format bare --public --file CentOS-7-x86_64-Everything-1804.iso Centos7.5_everything
# Make sure enough RAM is allocated 8G worked at time of testing
[root@localhost ~]# openstack flavor create m1.pass_8G --ram 8024 --disk 0 --vcpu 2
[root@localhost ~]# openstack flavor set m1.pass_8G --property "pci_passthrough:alias"="a1:2"
[root@localhost ~]# openstack flavor show m1.pass_8G
+----------------------------+--------------------------------------+
| Field | Value |
+----------------------------+--------------------------------------+
| OS-FLV-DISABLED:disabled | False |
| OS-FLV-EXT-DATA:ephemeral | 0 |
| access_project_ids | None |
| disk | 0 |
| id | d476fa2d-71d6-4686-b363-2b84baac3a3d |
| name | m1.pass_8G |
| os-flavor-access:is_public | True |
| properties | pci_passthrough:alias='a1:1' |
| ram | 8024 |
| rxtx_factor | 1.0 |
| swap | |
| vcpus | 2 |
+----------------------------+--------------------------------------+
[root@localhost ~]## to enable vfio drivers
root@localhost ~]# cat /etc/modprobe.d/vfio.conf
options vfio-pci ids=50c1:0003
root@localhost ~]# echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf
root@localhost ~]# reboot
# In base OS
root@localhost ~]# lspci -vvvvvvns 05:00
[root@localhost ~]# lspci -vvns 05:00
05:00.0 0480: 50c1:0003 (rev 01)
Subsystem: 50c1:0001
Physical Slot: 41
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B- DisINTx+
Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
Latency: 0
Interrupt: pin ? routed to IRQ 66
Region 0: Memory at df400000 (32-bit, non-prefetchable) [size=128K]
Region 2: Memory at df200000 (32-bit, non-prefetchable) [size=2M]
Region 4: Memory at df430000 (32-bit, non-prefetchable) [size=32K]
Region 5: Memory at df420000 (32-bit, non-prefetchable) [size=64K]
Capabilities: [40] Power Management version 3
Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=375mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
Status: D0 NoSoftRst+ PME-Enable- DSel=0 DScale=0 PME-
Capabilities: [c0] MSI: Enable+ Count=1/1 Maskable- 64bit+
Address: 00000000fee004d8 Data: 0000
Capabilities: [80] Express (v2) Endpoint, MSI 00
DevCap: MaxPayload 1024 bytes, PhantFunc 0, Latency L0s <64ns, L1 <1us
ExtTag+ AttnBtn- AttnInd- PwrInd- RBE+ FLReset- SlotPowerLimit 25.000W
DevCtl: Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
RlxdOrd- ExtTag+ PhantFunc- AuxPwr- NoSnoop+
MaxPayload 256 bytes, MaxReadReq 512 bytes
DevSta: CorrErr+ UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
LnkCap: Port #0, Speed 5GT/s, Width x4, ASPM L0s L1, Exit Latency L0s <1us, L1 <16us
ClockPM- Surprise- LLActRep- BwNot- ASPMOptComp+
LnkCtl: ASPM Disabled; RCB 64 bytes Disabled- CommClk-
ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
LnkSta: Speed 5GT/s, Width x4, TrErr- Train- SlotClk- DLActive- BWMgmt- ABWMgmt-
DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR-, OBFF Not Supported
DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR-, OBFF Disabled
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
LnkSta2: Current De-emphasis Level: -6dB, EqualizationComplete-, EqualizationPhase1-
EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
Capabilities: [100 v2] Advanced Error Reporting
UESta: DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
UEMsk: DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
UESvrt: DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
CESta: RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
CEMsk: RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
AERCap: First Error Pointer: 00, GenCap+ CGenEn- ChkCap+ ChkEn-
Capabilities: [200 v1] Virtual Channel
Caps: LPEVC=0 RefClk=100ns PATEntryBits=1
Arb: Fixed- WRR32- WRR64- WRR128-
Ctrl: ArbSelect=Fixed
Status: InProgress-
VC0: Caps: PATOffset=00 MaxTimeSlots=1 RejSnoopTrans-
Arb: Fixed- WRR32- WRR64- WRR128- TWRR128- WRR256-
Ctrl: Enable+ ID=0 ArbSelect=Fixed TC/VC=01
Status: NegoPending- InProgress-
Capabilities: [3a0 v1] Device Serial Number 00-00-00-00-00-00-00-01
Capabilities: [400 v1] Vendor Specific Information: ID=0020 Rev=3 Len=100 <?>
'''Kernel driver in use: vfio-pci'''
[root@localhost ~]# dmesg | grep 05:00
[ 0.223047] pci 0000:05:00.0: [50c1:0003] type 00 class 0x048000
[ 0.223063] pci 0000:05:00.0: reg 0x10: [mem 0xdf400000-0xdf41ffff]
[ 0.223077] pci 0000:05:00.0: reg 0x18: [mem 0xdf200000-0xdf3fffff]
[ 0.223091] pci 0000:05:00.0: reg 0x20: [mem 0xdf430000-0xdf437fff]
[ 0.223099] pci 0000:05:00.0: reg 0x24: [mem 0xdf420000-0xdf42ffff]
[ 0.223154] pci 0000:05:00.0: supports D1 D2
[ 0.223155] pci 0000:05:00.0: PME# supported from D0 D1 D2 D3hot D3cold
[ 0.515870] iommu: Adding device 0000:05:00.0 to group 1
[ 0.548264] pci 0000:05:00.0: Signaling PME through PCIe PME interrupt
[ 2036.852908] vfio-pci 0000:05:00.0: enabling device (0400 -> 0402)
[12533.193714] vfio-pci 0000:05:00.0: enabling device (0400 -> 0402)
[12751.678451] vfio-pci 0000:05:00.0: irq 66 for MSI/MSI-X #IRQ will be given Once the PCIe device is assigned in VM
[179033.494736] vfio-pci 0000:05:00.0: irq 66 for MSI/MSI-X