VScaler: Advantech PCIe passthrough - Revision history

Shanal at 17:41, 21 February 2020

2020-02-21T17:41:54Z

Shanal at 17:20, 21 February 2020

2020-02-21T17:20:51Z

Shanal at 17:01, 21 February 2020

2020-02-21T17:01:58Z

Shanal at 05:36, 26 November 2019

2019-11-26T05:36:13Z

Show changes

Shanal at 05:18, 26 November 2019

2019-11-26T05:18:04Z

Shanal at 04:58, 26 November 2019

2019-11-26T04:58:22Z

Shanal at 04:56, 26 November 2019

2019-11-26T04:56:35Z

Shanal at 21:58, 6 June 2019

2019-06-06T21:58:53Z

Shanal at 21:57, 6 June 2019

2019-06-06T21:57:14Z

Shanal at 21:24, 6 June 2019

2019-06-06T21:24:22Z

@@ Line 1: / Line 1: @@
-The setup :
+== The setup ==
 * Intel(R) Xeon(R) CPU E3-1225 v5 @ 3.30GHz
 * 16 GB RAM
 * Advantech Co. Ltd Device M30 card and fpga.
 We have setup an ALL in one scaler setup to passthrough the advantech card. The globals files for the installation is attached.
@@ Line 23: / Line 28: @@
 enable_haproxy: "no"
-:00.0 Multimedia controller: Advantech Co. Ltd Device 004b (rev 01)
+[root@localhost ~]#lspci -s 0b:00.0
 b:00.0 Multimedia controller: Device 50c1:0003 (rev 01)
 </syntaxhighlight>
-Advatech cards passthrough works with the following set of QEMU* version of rpms.
+Note : the other Qemu* rpm will end the system in Hang mode and restart after creating virtual machine.
 <syntaxhighlight>
@@ Line 59: / Line 73: @@
 </syntaxhighlight>
-<syntaxhighlight>
+The containers with Qemu* and QEMU-kvm* rpms are nova_libvirt , nova_compute , glance_api .
 <syntaxhighlight>
@@ Line 87: / Line 105: @@
 <syntaxhighlight>
 #check nova-scheduler , nova-api , nova-compute all have the setting
@@ Line 117: / Line 136: @@
 </syntaxhighlight>
-docker

@@ Line 289: / Line 289: @@
 	Kernel modules: vega3311_fpga
-[root@mediaflow-bioslab vega3311_fpgautil]# vega3311_cap_app list_dev
+</syntaxhighlight>
 DRV INF: SF: Got idcode 0xc2 0x20 0x1a
 Detected 1 chip(s).
 ==============================================
 Chip: 0
+	Device node name: /dev/vega3311_fpga_0
- Device node name: /dev/vega3311_fpga_0
+	PCI Bridge 0x08
+	PCI Bus:Slot.Func 09:00.0
- PCI Bridge 0x08
+	Vendor ID: 0x13fe	Device ID: 0x004b
+	Subsystem VendorID: 0x13fe	Subsystem DevID: 0x004b
- PCI Bus:Slot.Func 09:00.0
+	Serial Number: LKD0114896	Board ID: 9792A33111E
+	Irq: 18
- Vendor ID: 0x13fe Device ID: 0x004b
+	BAR Configuration:
+	Start              | Length
- Subsystem VendorID: 0x13fe Subsystem DevID: 0x004b
+x00000000de800000 | 000131072
+x00000000de400000 | 004194304
 ==============================================
+[root@mediaflow-bioslab scratch]#
-[root@mediaflow-bioslab vega3311_fpgautil]#
+[root@mediaflow-bioslab scratch]# vega3311_fpgautil list
+	[Card 0] Device id 0, PCIe bridge 8
 </syntaxhighlight>

← Older revision		Revision as of 17:01, 21 February 2020
Line 236:		Line 236:
	</syntaxhighlight>		</syntaxhighlight>

		+	<syntaxhighlight>
		+	[root@mediaflow-bioslab scratch]# tar -xvf vega_bqb_release_1_4_2_14270_x64.tar.bz2
		+	[root@mediaflow-bioslab scratch]# cd vega_bqb_release_1_4_2/vega3311_fpga_x64/
		+	driver/ example/ install.log install.sh Makefile sdk/ tools/ uninstall.sh
		+	[root@mediaflow-bioslab vega3311_fpga_x64]# ./install.sh
		+	Installation start.
		+	CentOS Linux release 7.4.1708 (Core)
		+	kernel-headers-3.10.0-693.el7.x86_64
		+	kernel-devel-3.10.0-693.el7.x86_64
		+	gcc-4.8.5-39.el7.x86_64
		+	Making sample applications...
		+	done.
		+	Installing sample applications...
		+	cp -f ./example/vega3311_capture_app/bin/vega3311_cap_app /usr/local/bin
		+	cp -f ./example/vega3311_capture_config/bin/vega3311_cap_config /usr/local/bin
		+	cp -f ./example/vega3311_transmit_config/bin/vega3311_trans_config /usr/local/bin
		+	cp -f ./tools/vega3311_fpgautil/bin/vega3311_fpgautil /usr/local/bin
		+	Installing header file and library...
		+	mkdir -p /usr/local/include/libvega_capture_api
		+	mkdir -p /usr/local/include/libvega_transmit_api
		+	cp -f ./driver/lib/libvega3311_fpga_drv.a /usr/local/lib
		+	cp -f ./driver/lib/libvega3311_fpga_drv.so /usr/local/lib
		+	cp -f ./sdk/vega3311_capture_api/lib/libvega3311_capture.a /usr/local/lib
		+	cp -f ./sdk/vega3311_capture_api/lib/libvega3311_capture.so /usr/local/lib
		+	cp -f ./sdk/vega3311_transmit_api/lib/libvega3311_transmit.a /usr/local/lib
		+	cp -f ./sdk/vega3311_transmit_api/lib/libvega3311_transmit.so /usr/local/lib
		+	cp -f ./driver/include/vega3311_fpga_drv.h /usr/local/include/libvega_capture_api
		+	cp -f ./sdk/vega3311_capture_api/include/VEGA3311_capture.h /usr/local/include/libvega_capture_api
		+	cp -f ./sdk/vega3311_capture_api/include/VEGA3311_cap_types.h /usr/local/include/libvega_capture_api
		+	cp -f ./sdk/vega3311_transmit_api/include/VEGA3311_transmit.h /usr/local/include/libvega_transmit_api
		+	cp -f ./sdk/vega3311_transmit_api/include/VEGA3311_trans_types.h /usr/local/include/libvega_transmit_api
		+	Clean the package...
		+	done.
		+	Making and loading device driver...done.
		+	Installation completed.
		+
		+	[root@mediaflow-bioslab scratch]# lspci -vs 09:00.0
		+	09:00.0 Multimedia controller: Advantech Co. Ltd Device 004b (rev 01)
		+	Subsystem: Advantech Co. Ltd Device 004b
		+	Physical Slot: 1
		+	Flags: bus master, fast devsel, latency 0, IRQ 18
		+	Memory at de800000 (32-bit, non-prefetchable) [size=128K]
		+	Memory at de400000 (32-bit, non-prefetchable) [size=4M]
		+	Memory at 3fff800000 (64-bit, prefetchable) [size=4M]
		+	Capabilities: [80] Power Management version 3
		+	Capabilities: [90] MSI: Enable- Count=1/1 Maskable- 64bit+
		+	Capabilities: [b0] MSI-X: Enable+ Count=32 Masked-
		+	Capabilities: [c0] Express Endpoint, MSI 00
		+	Capabilities: [100] Advanced Error Reporting
		+	Capabilities: [340] Vendor Specific Information: ID=0001 Rev=0 Len=02c <?>
		+	Kernel driver in use: vega3311_fpga
		+	Kernel modules: vega3311_fpga
		+
		+	[16:13] Shanal Tiwari
		+
		+
		+	[root@mediaflow-bioslab vega3311_fpgautil]# vega3311_cap_app list_dev
		+
		+	DRV INF: SF: Got idcode 0xc2 0x20 0x1a
		+
		+	Detected 1 chip(s).
		+
		+	==============================================
		+
		+	Chip: 0
		+
		+	Device node name: /dev/vega3311_fpga_0
		+
		+	PCI Bridge 0x08
		+
		+	PCI Bus:Slot.Func 09:00.0
		+
		+	Vendor ID: 0x13fe Device ID: 0x004b
		+
		+	Subsystem VendorID: 0x13fe Subsystem DevID: 0x004b
		+
		+	Serial Number: LKD0114896 Board ID: 9792A33111E
		+
		+	Irq: 18
		+
		+	BAR Configuration:
		+
		+	Start \| Length
		+
		+	0x00000000de800000 \| 000131072
		+
		+	0x00000000de400000 \| 004194304
		+
		+	==============================================
		+
		+	[root@mediaflow-bioslab vega3311_fpgautil]#
		+
		+	[16:13] Shanal Tiwari
		+
		+
		+	[root@mediaflow-bioslab vega3311_fpgautil]# vega3311_fpgautil list
		+
		+	[Card 0] Device id 0, PCIe bridge 8
		+
		+	[16:14] Shanal Tiwari
		+
		+
		+	[root@mediaflow-bioslab vega3311_fpgautil]# vega3311_fpgautil qpcie 0
		+
		+	DRV INF: SF: Got idcode 0xc2 0x20 0x1a
		+
		+	==============================================
		+
		+	Chip: 0
		+
		+	Device node name: /dev/vega3311_fpga_0
		+
		+	PCI Bridge 0x08
		+
		+	PCI Bus:Slot.Func 09:00.0
		+
		+	Vendor ID: 0x13fe Device ID: 0x004b
		+
		+	Subsystem VendorID: 0x13fe Subsystem DevID: 0x004b
		+
		+	Class: 0x00048000
		+
		+	Header Type: 0 Irq: 18
		+
		+	BAR Configuration:
		+
		+	Start \| Length \| Flags
		+
		+	0x00000000de800000 \| 000131072 \| 0x00040200
		+
		+	0x00000000de400000 \| 004194304 \| 0x00040200
		+
		+	0x0000003fff800000 \| 004194304 \| 0x0014220c
		+
		+	bootup image: 1st image (3G SDI)
		+
		+	working image: 1st image (3G SDI)
		+
		+	==============================================
		+
		+
		+	</syntaxhighlight>
	Compling drivers for the fgpa (history back up from Roy VM)		Compling drivers for the fgpa (history back up from Roy VM)

@@ Line 36: / Line 36: @@
 qemu-kvm-common-ev-2.6.0-28.el7.centos_3.6.1.x86_64.rpm  qemu-kvm-tools-ev-2.6.0-28.el7.centos_3.6.1.x86_64.rpm
 </syntaxhighlight>
 <syntaxhighlight>

@@ Line 22: / Line 22: @@
 enable_cinder: "yes"
 enable_haproxy: "no"
 </syntaxhighlight>

← Older revision		Revision as of 04:56, 26 November 2019
Line 1:		Line 1:
		+	The setup :
		+
		+	* Intel(R) Xeon(R) CPU E3-1225 v5 @ 3.30GHz
		+	* 16 GB RAM
		+	* Advantech Co. Ltd Device M30 card and fpga.
		+
		+	We have setup an ALL in one scaler setup to passthrough the advantech card. The globals files for the installation is attached.
		+
		+	<syntaxhighlight>
		+	[root@localhost ~]# cat /etc/kolla/globals.yml \| grep -v '#'
		+	---
		+	kolla_base_distro: "centos"
		+	kolla_install_type: "binary"
		+	openstack_release: "6.0.1"
		+	node_custom_config: "/etc/kolla/config"
		+	kolla_internal_vip_address: "185.93.28.189"
		+	kolla_external_vip_address: "{{ kolla_internal_vip_address }}"
		+	docker_registry: "registry.vscaler.com:5000"
		+	network_interface: "eno1"
		+	neutron_external_interface: "eno2"
		+	enable_ceph: "yes"
		+	enable_cinder: "yes"
		+	enable_haproxy: "no"
		+	</syntaxhighlight>
		+
		+
		+	Advatech cards passthrough works with the following set of QEMU* version of rpms.
		+
		+	<syntaxhighlight>
		+	[root@localhost ~]# ls qemu/
		+	qemu-img-ev-2.6.0-28.el7.centos_3.6.1.x86_64.rpm qemu-kvm-ev-2.6.0-28.el7.centos_3.6.1.x86_64.rpm
		+	qemu-kvm-common-ev-2.6.0-28.el7.centos_3.6.1.x86_64.rpm qemu-kvm-tools-ev-2.6.0-28.el7.centos_3.6.1.x86_64.rpm
		+
		+
		+
		+	</syntaxhighlight>
		+
		+
	<syntaxhighlight>		<syntaxhighlight>
	[root@localhost ~]# vi /etc/kolla/config/nova.conf		[root@localhost ~]# vi /etc/kolla/config/nova.conf

@@ Line 1: / Line 1: @@
 <syntaxhighlight>
 [root@localhost ~]# cat /etc/kolla/config/nova.conf
 [filter_scheduler]
@@ Line 12: / Line 13: @@
 passthrough_whitelist = { "address": "0000:05:00.0" }
 alias={"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"}
 </syntaxhighlight>

@@ Line 13: / Line 13: @@
 alias={"vendor_id":"50c1","product_id":"0003","device_type":"type-PCI","name":"a1"}
 </syntaxhighlight>
-<syntaxhighlight>
+</syntaxhighlight>
 [root@localhost ~]# openstack image create --disk-format raw --container-format bare --public  --file CentOS-7-x86_64-Everything-1804.iso Centos7.5_everything
 [root@localhost ~]# openstack flavor show  m1.pass_8G
 +----------------------------+--------------------------------------+
@@ Line 109: / Line 143: @@
          Capabilities: [400 v1] Vendor Specific Information: ID=0020 Rev=3 Len=100 <?>
          '''Kernel driver in use: vfio-pci'''
 </syntaxhighlight>