1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Char device for device raw access
   4 *
   5 * Copyright (C) 2005-2007  Kristian Hoegsberg <krh@bitplanet.net>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
   6 */
   7
   8#include <linux/bug.h>
   9#include <linux/compat.h>
  10#include <linux/delay.h>
  11#include <linux/device.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/err.h>
  14#include <linux/errno.h>
  15#include <linux/firewire.h>
  16#include <linux/firewire-cdev.h>
 
  17#include <linux/irqflags.h>
  18#include <linux/jiffies.h>
  19#include <linux/kernel.h>
  20#include <linux/kref.h>
  21#include <linux/mm.h>
  22#include <linux/module.h>
  23#include <linux/mutex.h>
  24#include <linux/poll.h>
  25#include <linux/sched.h> /* required for linux/wait.h */
  26#include <linux/slab.h>
  27#include <linux/spinlock.h>
  28#include <linux/string.h>
  29#include <linux/time.h>
  30#include <linux/uaccess.h>
  31#include <linux/vmalloc.h>
  32#include <linux/wait.h>
  33#include <linux/workqueue.h>
  34
  35
  36#include "core.h"
  37#include <trace/events/firewire.h>
  38
  39#include "packet-header-definitions.h"
  40
  41/*
  42 * ABI version history is documented in linux/firewire-cdev.h.
  43 */
  44#define FW_CDEV_KERNEL_VERSION			5
  45#define FW_CDEV_VERSION_EVENT_REQUEST2		4
  46#define FW_CDEV_VERSION_ALLOCATE_REGION_END	4
  47#define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW	5
  48#define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP	6
  49
  50struct client {
  51	u32 version;
  52	struct fw_device *device;
  53
  54	spinlock_t lock;
  55	bool in_shutdown;
  56	struct xarray resource_xa;
  57	struct list_head event_list;
  58	wait_queue_head_t wait;
  59	wait_queue_head_t tx_flush_wait;
  60	u64 bus_reset_closure;
  61
  62	struct fw_iso_context *iso_context;
  63	u64 iso_closure;
  64	struct fw_iso_buffer buffer;
  65	unsigned long vm_start;
  66	bool buffer_is_mapped;
  67
  68	struct list_head phy_receiver_link;
  69	u64 phy_receiver_closure;
  70
  71	struct list_head link;
  72	struct kref kref;
  73};
  74
  75static inline void client_get(struct client *client)
  76{
  77	kref_get(&client->kref);
  78}
  79
  80static void client_release(struct kref *kref)
  81{
  82	struct client *client = container_of(kref, struct client, kref);
  83
  84	fw_device_put(client->device);
  85	kfree(client);
  86}
  87
  88static void client_put(struct client *client)
  89{
  90	kref_put(&client->kref, client_release);
  91}
  92
  93struct client_resource;
  94typedef void (*client_resource_release_fn_t)(struct client *,
  95					     struct client_resource *);
  96struct client_resource {
  97	client_resource_release_fn_t release;
  98	int handle;
  99};
 100
 101struct address_handler_resource {
 102	struct client_resource resource;
 103	struct fw_address_handler handler;
 104	__u64 closure;
 105	struct client *client;
 106};
 107
 108struct outbound_transaction_resource {
 109	struct client_resource resource;
 110	struct fw_transaction transaction;
 111};
 112
 113struct inbound_transaction_resource {
 114	struct client_resource resource;
 115	struct fw_card *card;
 116	struct fw_request *request;
 117	bool is_fcp;
 118	void *data;
 119	size_t length;
 120};
 121
 122struct descriptor_resource {
 123	struct client_resource resource;
 124	struct fw_descriptor descriptor;
 125	u32 data[];
 126};
 127
 128struct iso_resource {
 129	struct client_resource resource;
 130	struct client *client;
 131	/* Schedule work and access todo only with client->lock held. */
 132	struct delayed_work work;
 133	enum {ISO_RES_ALLOC, ISO_RES_REALLOC, ISO_RES_DEALLOC,
 134	      ISO_RES_ALLOC_ONCE, ISO_RES_DEALLOC_ONCE,} todo;
 135	int generation;
 136	u64 channels;
 137	s32 bandwidth;
 138	struct iso_resource_event *e_alloc, *e_dealloc;
 139};
 140
 141static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource)
 142{
 143	return container_of(resource, struct address_handler_resource, resource);
 144}
 145
 146static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource)
 147{
 148	return container_of(resource, struct inbound_transaction_resource, resource);
 149}
 150
 151static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource)
 152{
 153	return container_of(resource, struct descriptor_resource, resource);
 154}
 155
 156static struct iso_resource *to_iso_resource(struct client_resource *resource)
 157{
 158	return container_of(resource, struct iso_resource, resource);
 159}
 160
 161static void release_iso_resource(struct client *, struct client_resource *);
 162
 163static int is_iso_resource(const struct client_resource *resource)
 164{
 165	return resource->release == release_iso_resource;
 166}
 167
 168static void release_transaction(struct client *client,
 169				struct client_resource *resource);
 170
 171static int is_outbound_transaction_resource(const struct client_resource *resource)
 172{
 173	return resource->release == release_transaction;
 174}
 175
 176static void schedule_iso_resource(struct iso_resource *r, unsigned long delay)
 177{
 178	client_get(r->client);
 179	if (!queue_delayed_work(fw_workqueue, &r->work, delay))
 180		client_put(r->client);
 181}
 182
 
 
 
 
 
 
 
 183/*
 184 * dequeue_event() just kfree()'s the event, so the event has to be
 185 * the first field in a struct XYZ_event.
 186 */
 187struct event {
 188	struct { void *data; size_t size; } v[2];
 189	struct list_head link;
 190};
 191
 192struct bus_reset_event {
 193	struct event event;
 194	struct fw_cdev_event_bus_reset reset;
 195};
 196
 197struct outbound_transaction_event {
 198	struct event event;
 199	struct client *client;
 200	struct outbound_transaction_resource r;
 201	union {
 202		struct fw_cdev_event_response without_tstamp;
 203		struct fw_cdev_event_response2 with_tstamp;
 204	} rsp;
 205};
 206
 207struct inbound_transaction_event {
 208	struct event event;
 209	union {
 210		struct fw_cdev_event_request request;
 211		struct fw_cdev_event_request2 request2;
 212		struct fw_cdev_event_request3 with_tstamp;
 213	} req;
 214};
 215
 216struct iso_interrupt_event {
 217	struct event event;
 218	struct fw_cdev_event_iso_interrupt interrupt;
 219};
 220
 221struct iso_interrupt_mc_event {
 222	struct event event;
 223	struct fw_cdev_event_iso_interrupt_mc interrupt;
 224};
 225
 226struct iso_resource_event {
 227	struct event event;
 228	struct fw_cdev_event_iso_resource iso_resource;
 229};
 230
 231struct outbound_phy_packet_event {
 232	struct event event;
 233	struct client *client;
 234	struct fw_packet p;
 235	union {
 236		struct fw_cdev_event_phy_packet without_tstamp;
 237		struct fw_cdev_event_phy_packet2 with_tstamp;
 238	} phy_packet;
 239};
 240
 241struct inbound_phy_packet_event {
 242	struct event event;
 243	union {
 244		struct fw_cdev_event_phy_packet without_tstamp;
 245		struct fw_cdev_event_phy_packet2 with_tstamp;
 246	} phy_packet;
 247};
 248
 249#ifdef CONFIG_COMPAT
 250static void __user *u64_to_uptr(u64 value)
 251{
 252	if (in_compat_syscall())
 253		return compat_ptr(value);
 254	else
 255		return (void __user *)(unsigned long)value;
 256}
 257
 258static u64 uptr_to_u64(void __user *ptr)
 259{
 260	if (in_compat_syscall())
 261		return ptr_to_compat(ptr);
 262	else
 263		return (u64)(unsigned long)ptr;
 264}
 265#else
 266static inline void __user *u64_to_uptr(u64 value)
 267{
 268	return (void __user *)(unsigned long)value;
 269}
 270
 271static inline u64 uptr_to_u64(void __user *ptr)
 272{
 273	return (u64)(unsigned long)ptr;
 274}
 275#endif /* CONFIG_COMPAT */
 276
 277static int fw_device_op_open(struct inode *inode, struct file *file)
 278{
 279	struct fw_device *device;
 280	struct client *client;
 281
 282	device = fw_device_get_by_devt(inode->i_rdev);
 283	if (device == NULL)
 284		return -ENODEV;
 285
 286	if (fw_device_is_shutdown(device)) {
 287		fw_device_put(device);
 288		return -ENODEV;
 289	}
 290
 291	client = kzalloc(sizeof(*client), GFP_KERNEL);
 292	if (client == NULL) {
 293		fw_device_put(device);
 294		return -ENOMEM;
 295	}
 296
 297	client->device = device;
 298	spin_lock_init(&client->lock);
 299	xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);
 300	INIT_LIST_HEAD(&client->event_list);
 301	init_waitqueue_head(&client->wait);
 302	init_waitqueue_head(&client->tx_flush_wait);
 303	INIT_LIST_HEAD(&client->phy_receiver_link);
 304	INIT_LIST_HEAD(&client->link);
 305	kref_init(&client->kref);
 306
 307	file->private_data = client;
 308
 309	return nonseekable_open(inode, file);
 310}
 311
 312static void queue_event(struct client *client, struct event *event,
 313			void *data0, size_t size0, void *data1, size_t size1)
 314{
 
 
 315	event->v[0].data = data0;
 316	event->v[0].size = size0;
 317	event->v[1].data = data1;
 318	event->v[1].size = size1;
 319
 320	scoped_guard(spinlock_irqsave, &client->lock) {
 321		if (client->in_shutdown)
 322			kfree(event);
 323		else
 324			list_add_tail(&event->link, &client->event_list);
 325	}
 326
 327	wake_up_interruptible(&client->wait);
 328}
 329
 330static int dequeue_event(struct client *client,
 331			 char __user *buffer, size_t count)
 332{
 333	struct event *event;
 334	size_t size, total;
 335	int i, ret;
 336
 337	ret = wait_event_interruptible(client->wait,
 338			!list_empty(&client->event_list) ||
 339			fw_device_is_shutdown(client->device));
 340	if (ret < 0)
 341		return ret;
 342
 343	if (list_empty(&client->event_list) &&
 344		       fw_device_is_shutdown(client->device))
 345		return -ENODEV;
 346
 347	scoped_guard(spinlock_irq, &client->lock) {
 348		event = list_first_entry(&client->event_list, struct event, link);
 349		list_del(&event->link);
 350	}
 351
 352	total = 0;
 353	for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) {
 354		size = min(event->v[i].size, count - total);
 355		if (copy_to_user(buffer + total, event->v[i].data, size)) {
 356			ret = -EFAULT;
 357			goto out;
 358		}
 359		total += size;
 360	}
 361	ret = total;
 362
 363 out:
 364	kfree(event);
 365
 366	return ret;
 367}
 368
 369static ssize_t fw_device_op_read(struct file *file, char __user *buffer,
 370				 size_t count, loff_t *offset)
 371{
 372	struct client *client = file->private_data;
 373
 374	return dequeue_event(client, buffer, count);
 375}
 376
 377static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,
 378				 struct client *client)
 379{
 380	struct fw_card *card = client->device->card;
 381
 382	guard(spinlock_irq)(&card->lock);
 383
 384	event->closure	     = client->bus_reset_closure;
 385	event->type          = FW_CDEV_EVENT_BUS_RESET;
 386	event->generation    = client->device->generation;
 387	event->node_id       = client->device->node_id;
 388	event->local_node_id = card->local_node->node_id;
 389	event->bm_node_id    = card->bm_node_id;
 390	event->irm_node_id   = card->irm_node->node_id;
 391	event->root_node_id  = card->root_node->node_id;
 
 
 392}
 393
 394static void for_each_client(struct fw_device *device,
 395			    void (*callback)(struct client *client))
 396{
 397	struct client *c;
 398
 399	guard(mutex)(&device->client_list_mutex);
 400
 401	list_for_each_entry(c, &device->client_list, link)
 402		callback(c);
 
 
 
 
 
 
 
 
 403}
 404
 405static void queue_bus_reset_event(struct client *client)
 406{
 407	struct bus_reset_event *e;
 408	struct client_resource *resource;
 409	unsigned long index;
 410
 411	e = kzalloc(sizeof(*e), GFP_KERNEL);
 412	if (e == NULL)
 413		return;
 414
 415	fill_bus_reset_event(&e->reset, client);
 416
 417	queue_event(client, &e->event,
 418		    &e->reset, sizeof(e->reset), NULL, 0);
 419
 420	guard(spinlock_irq)(&client->lock);
 421
 422	xa_for_each(&client->resource_xa, index, resource) {
 423		if (is_iso_resource(resource))
 424			schedule_iso_resource(to_iso_resource(resource), 0);
 425	}
 426}
 427
 428void fw_device_cdev_update(struct fw_device *device)
 429{
 430	for_each_client(device, queue_bus_reset_event);
 431}
 432
 433static void wake_up_client(struct client *client)
 434{
 435	wake_up_interruptible(&client->wait);
 436}
 437
 438void fw_device_cdev_remove(struct fw_device *device)
 439{
 440	for_each_client(device, wake_up_client);
 441}
 442
 443union ioctl_arg {
 444	struct fw_cdev_get_info			get_info;
 445	struct fw_cdev_send_request		send_request;
 446	struct fw_cdev_allocate			allocate;
 447	struct fw_cdev_deallocate		deallocate;
 448	struct fw_cdev_send_response		send_response;
 449	struct fw_cdev_initiate_bus_reset	initiate_bus_reset;
 450	struct fw_cdev_add_descriptor		add_descriptor;
 451	struct fw_cdev_remove_descriptor	remove_descriptor;
 452	struct fw_cdev_create_iso_context	create_iso_context;
 453	struct fw_cdev_queue_iso		queue_iso;
 454	struct fw_cdev_start_iso		start_iso;
 455	struct fw_cdev_stop_iso			stop_iso;
 456	struct fw_cdev_get_cycle_timer		get_cycle_timer;
 457	struct fw_cdev_allocate_iso_resource	allocate_iso_resource;
 458	struct fw_cdev_send_stream_packet	send_stream_packet;
 459	struct fw_cdev_get_cycle_timer2		get_cycle_timer2;
 460	struct fw_cdev_send_phy_packet		send_phy_packet;
 461	struct fw_cdev_receive_phy_packets	receive_phy_packets;
 462	struct fw_cdev_set_iso_channels		set_iso_channels;
 463	struct fw_cdev_flush_iso		flush_iso;
 464};
 465
 466static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
 467{
 468	struct fw_cdev_get_info *a = &arg->get_info;
 469	struct fw_cdev_event_bus_reset bus_reset;
 470	unsigned long ret = 0;
 471
 472	client->version = a->version;
 473	a->version = FW_CDEV_KERNEL_VERSION;
 474	a->card = client->device->card->index;
 475
 476	scoped_guard(rwsem_read, &fw_device_rwsem) {
 477		if (a->rom != 0) {
 478			size_t want = a->rom_length;
 479			size_t have = client->device->config_rom_length * 4;
 480
 481			ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom,
 482					   min(want, have));
 483			if (ret != 0)
 484				return -EFAULT;
 485		}
 486		a->rom_length = client->device->config_rom_length * 4;
 487	}
 
 
 
 488
 489	guard(mutex)(&client->device->client_list_mutex);
 
 
 
 490
 491	client->bus_reset_closure = a->bus_reset_closure;
 492	if (a->bus_reset != 0) {
 493		fill_bus_reset_event(&bus_reset, client);
 494		/* unaligned size of bus_reset is 36 bytes */
 495		ret = copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36);
 496	}
 497	if (ret == 0 && list_empty(&client->link))
 498		list_add_tail(&client->link, &client->device->client_list);
 499
 
 
 500	return ret ? -EFAULT : 0;
 501}
 502
 503static int add_client_resource(struct client *client, struct client_resource *resource,
 504			       gfp_t gfp_mask)
 505{
 
 
 506	int ret;
 507
 508	scoped_guard(spinlock_irqsave, &client->lock) {
 509		u32 index;
 
 510
 511		if (client->in_shutdown) {
 512			ret = -ECANCELED;
 513		} else {
 514			if (gfpflags_allow_blocking(gfp_mask)) {
 515				ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,
 516					       GFP_NOWAIT);
 517			} else {
 518				ret = xa_alloc_bh(&client->resource_xa, &index, resource,
 519						  xa_limit_32b, GFP_NOWAIT);
 520			}
 521		}
 522		if (ret >= 0) {
 523			resource->handle = index;
 524			client_get(client);
 525			if (is_iso_resource(resource))
 526				schedule_iso_resource(to_iso_resource(resource), 0);
 527		}
 528	}
 529
 
 
 
 
 530	return ret < 0 ? ret : 0;
 531}
 532
 533static int release_client_resource(struct client *client, u32 handle,
 534				   client_resource_release_fn_t release,
 535				   struct client_resource **return_resource)
 536{
 537	unsigned long index = handle;
 538	struct client_resource *resource;
 539
 540	scoped_guard(spinlock_irq, &client->lock) {
 541		if (client->in_shutdown)
 542			return -EINVAL;
 543
 544		resource = xa_load(&client->resource_xa, index);
 545		if (!resource || resource->release != release)
 546			return -EINVAL;
 
 547
 548		xa_erase(&client->resource_xa, handle);
 549	}
 550
 551	if (return_resource)
 552		*return_resource = resource;
 553	else
 554		resource->release(client, resource);
 555
 556	client_put(client);
 557
 558	return 0;
 559}
 560
 561static void release_transaction(struct client *client,
 562				struct client_resource *resource)
 563{
 564}
 565
 566static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp,
 567				 u32 response_tstamp, void *payload, size_t length, void *data)
 568{
 569	struct outbound_transaction_event *e = data;
 
 570	struct client *client = e->client;
 571	unsigned long index = e->r.resource.handle;
 572
 573	scoped_guard(spinlock_irqsave, &client->lock) {
 574		xa_erase(&client->resource_xa, index);
 575		if (client->in_shutdown)
 576			wake_up(&client->tx_flush_wait);
 577	}
 578
 579	switch (e->rsp.without_tstamp.type) {
 580	case FW_CDEV_EVENT_RESPONSE:
 581	{
 582		struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
 583
 584		if (length < rsp->length)
 585			rsp->length = length;
 586		if (rcode == RCODE_COMPLETE)
 587			memcpy(rsp->data, payload, rsp->length);
 588
 589		rsp->rcode = rcode;
 590
 591		// In the case that sizeof(*rsp) doesn't align with the position of the
 592		// data, and the read is short, preserve an extra copy of the data
 593		// to stay compatible with a pre-2.6.27 bug.  Since the bug is harmless
 594		// for short reads and some apps depended on it, this is both safe
 595		// and prudent for compatibility.
 596		if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data))
 597			queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length);
 598		else
 599			queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
 600
 601		break;
 602	}
 603	case FW_CDEV_EVENT_RESPONSE2:
 604	{
 605		struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
 606
 607		if (length < rsp->length)
 608			rsp->length = length;
 609		if (rcode == RCODE_COMPLETE)
 610			memcpy(rsp->data, payload, rsp->length);
 611
 612		rsp->rcode = rcode;
 613		rsp->request_tstamp = request_tstamp;
 614		rsp->response_tstamp = response_tstamp;
 615
 616		queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
 
 617
 618		break;
 619	}
 620	default:
 621		WARN_ON(1);
 622		break;
 623	}
 
 
 
 
 
 
 
 624
 625	// Drop the xarray's reference.
 626	client_put(client);
 627}
 628
 629static int init_request(struct client *client,
 630			struct fw_cdev_send_request *request,
 631			int destination_id, int speed)
 632{
 633	struct outbound_transaction_event *e;
 634	void *payload;
 635	int ret;
 636
 637	if (request->tcode != TCODE_STREAM_DATA &&
 638	    (request->length > 4096 || request->length > 512 << speed))
 639		return -EIO;
 640
 641	if (request->tcode == TCODE_WRITE_QUADLET_REQUEST &&
 642	    request->length < 4)
 643		return -EINVAL;
 644
 645	e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL);
 646	if (e == NULL)
 647		return -ENOMEM;
 648	e->client = client;
 649
 650	if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
 651		struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
 652
 653		rsp->type = FW_CDEV_EVENT_RESPONSE;
 654		rsp->length = request->length;
 655		rsp->closure = request->closure;
 656		payload = rsp->data;
 657	} else {
 658		struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
 659
 660		rsp->type = FW_CDEV_EVENT_RESPONSE2;
 661		rsp->length = request->length;
 662		rsp->closure = request->closure;
 663		payload = rsp->data;
 664	}
 665
 666	if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) {
 
 
 667		ret = -EFAULT;
 668		goto failed;
 669	}
 670
 671	e->r.resource.release = release_transaction;
 672	ret = add_client_resource(client, &e->r.resource, GFP_KERNEL);
 673	if (ret < 0)
 674		goto failed;
 675
 676	fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode,
 677				    destination_id, request->generation, speed, request->offset,
 678				    payload, request->length, complete_transaction, e);
 
 679	return 0;
 680
 681 failed:
 682	kfree(e);
 683
 684	return ret;
 685}
 686
 687static int ioctl_send_request(struct client *client, union ioctl_arg *arg)
 688{
 689	switch (arg->send_request.tcode) {
 690	case TCODE_WRITE_QUADLET_REQUEST:
 691	case TCODE_WRITE_BLOCK_REQUEST:
 692	case TCODE_READ_QUADLET_REQUEST:
 693	case TCODE_READ_BLOCK_REQUEST:
 694	case TCODE_LOCK_MASK_SWAP:
 695	case TCODE_LOCK_COMPARE_SWAP:
 696	case TCODE_LOCK_FETCH_ADD:
 697	case TCODE_LOCK_LITTLE_ADD:
 698	case TCODE_LOCK_BOUNDED_ADD:
 699	case TCODE_LOCK_WRAP_ADD:
 700	case TCODE_LOCK_VENDOR_DEPENDENT:
 701		break;
 702	default:
 703		return -EINVAL;
 704	}
 705
 706	return init_request(client, &arg->send_request, client->device->node_id,
 707			    client->device->max_speed);
 708}
 709
 
 
 
 
 
 710static void release_request(struct client *client,
 711			    struct client_resource *resource)
 712{
 713	struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource);
 
 714
 715	if (r->is_fcp)
 716		fw_request_put(r->request);
 717	else
 718		fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR);
 719
 720	fw_card_put(r->card);
 721	kfree(r);
 722}
 723
 724static void handle_request(struct fw_card *card, struct fw_request *request,
 725			   int tcode, int destination, int source,
 726			   int generation, unsigned long long offset,
 727			   void *payload, size_t length, void *callback_data)
 728{
 729	struct address_handler_resource *handler = callback_data;
 730	bool is_fcp = is_in_fcp_region(offset, length);
 731	struct inbound_transaction_resource *r;
 732	struct inbound_transaction_event *e;
 733	size_t event_size0;
 
 734	int ret;
 735
 736	/* card may be different from handler->client->device->card */
 737	fw_card_get(card);
 738
 739	// Extend the lifetime of data for request so that its payload is safely accessible in
 740	// the process context for the client.
 741	if (is_fcp)
 742		fw_request_get(request);
 743
 744	r = kmalloc(sizeof(*r), GFP_ATOMIC);
 745	e = kmalloc(sizeof(*e), GFP_ATOMIC);
 746	if (r == NULL || e == NULL)
 747		goto failed;
 748
 749	r->card    = card;
 750	r->request = request;
 751	r->is_fcp  = is_fcp;
 752	r->data    = payload;
 753	r->length  = length;
 754
 
 
 
 
 
 
 
 
 
 
 
 
 755	r->resource.release = release_request;
 756	ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC);
 757	if (ret < 0)
 758		goto failed;
 759
 760	if (handler->client->version < FW_CDEV_VERSION_EVENT_REQUEST2) {
 761		struct fw_cdev_event_request *req = &e->req.request;
 762
 763		if (tcode & 0x10)
 764			tcode = TCODE_LOCK_REQUEST;
 765
 766		req->type	= FW_CDEV_EVENT_REQUEST;
 767		req->tcode	= tcode;
 768		req->offset	= offset;
 769		req->length	= length;
 770		req->handle	= r->resource.handle;
 771		req->closure	= handler->closure;
 772		event_size0	= sizeof(*req);
 773	} else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
 774		struct fw_cdev_event_request2 *req = &e->req.request2;
 775
 776		req->type	= FW_CDEV_EVENT_REQUEST2;
 777		req->tcode	= tcode;
 778		req->offset	= offset;
 779		req->source_node_id = source;
 780		req->destination_node_id = destination;
 781		req->card	= card->index;
 782		req->generation	= generation;
 783		req->length	= length;
 784		req->handle	= r->resource.handle;
 785		req->closure	= handler->closure;
 786		event_size0	= sizeof(*req);
 787	} else {
 788		struct fw_cdev_event_request3 *req = &e->req.with_tstamp;
 789
 790		req->type	= FW_CDEV_EVENT_REQUEST3;
 791		req->tcode	= tcode;
 792		req->offset	= offset;
 793		req->source_node_id = source;
 794		req->destination_node_id = destination;
 795		req->card	= card->index;
 796		req->generation	= generation;
 797		req->length	= length;
 798		req->handle	= r->resource.handle;
 799		req->closure	= handler->closure;
 800		req->tstamp	= fw_request_get_timestamp(request);
 801		event_size0	= sizeof(*req);
 802	}
 803
 804	queue_event(handler->client, &e->event,
 805		    &e->req, event_size0, r->data, length);
 806	return;
 807
 808 failed:
 809	kfree(r);
 810	kfree(e);
 
 811
 812	if (!is_fcp)
 813		fw_send_response(card, request, RCODE_CONFLICT_ERROR);
 814	else
 815		fw_request_put(request);
 816
 817	fw_card_put(card);
 818}
 819
 820static void release_address_handler(struct client *client,
 821				    struct client_resource *resource)
 822{
 823	struct address_handler_resource *r = to_address_handler_resource(resource);
 
 824
 825	fw_core_remove_address_handler(&r->handler);
 826	kfree(r);
 827}
 828
 829static int ioctl_allocate(struct client *client, union ioctl_arg *arg)
 830{
 831	struct fw_cdev_allocate *a = &arg->allocate;
 832	struct address_handler_resource *r;
 833	struct fw_address_region region;
 834	int ret;
 835
 836	r = kmalloc(sizeof(*r), GFP_KERNEL);
 837	if (r == NULL)
 838		return -ENOMEM;
 839
 840	region.start = a->offset;
 841	if (client->version < FW_CDEV_VERSION_ALLOCATE_REGION_END)
 842		region.end = a->offset + a->length;
 843	else
 844		region.end = a->region_end;
 845
 846	r->handler.length           = a->length;
 847	r->handler.address_callback = handle_request;
 848	r->handler.callback_data    = r;
 849	r->closure   = a->closure;
 850	r->client    = client;
 851
 852	ret = fw_core_add_address_handler(&r->handler, &region);
 853	if (ret < 0) {
 854		kfree(r);
 855		return ret;
 856	}
 857	a->offset = r->handler.offset;
 858
 859	r->resource.release = release_address_handler;
 860	ret = add_client_resource(client, &r->resource, GFP_KERNEL);
 861	if (ret < 0) {
 862		release_address_handler(client, &r->resource);
 863		return ret;
 864	}
 865	a->handle = r->resource.handle;
 866
 867	return 0;
 868}
 869
 870static int ioctl_deallocate(struct client *client, union ioctl_arg *arg)
 871{
 872	return release_client_resource(client, arg->deallocate.handle,
 873				       release_address_handler, NULL);
 874}
 875
 876static int ioctl_send_response(struct client *client, union ioctl_arg *arg)
 877{
 878	struct fw_cdev_send_response *a = &arg->send_response;
 879	struct client_resource *resource;
 880	struct inbound_transaction_resource *r;
 881	int ret = 0;
 882
 883	if (release_client_resource(client, a->handle,
 884				    release_request, &resource) < 0)
 885		return -EINVAL;
 886
 887	r = to_inbound_transaction_resource(resource);
 888	if (r->is_fcp) {
 889		fw_request_put(r->request);
 890		goto out;
 891	}
 892
 893	if (a->length != fw_get_response_length(r->request)) {
 894		ret = -EINVAL;
 895		fw_request_put(r->request);
 896		goto out;
 897	}
 898	if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) {
 899		ret = -EFAULT;
 900		fw_request_put(r->request);
 901		goto out;
 902	}
 903	fw_send_response(r->card, r->request, a->rcode);
 904 out:
 905	fw_card_put(r->card);
 906	kfree(r);
 907
 908	return ret;
 909}
 910
 911static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg)
 912{
 913	fw_schedule_bus_reset(client->device->card, true,
 914			arg->initiate_bus_reset.type == FW_CDEV_SHORT_RESET);
 915	return 0;
 916}
 917
 918static void release_descriptor(struct client *client,
 919			       struct client_resource *resource)
 920{
 921	struct descriptor_resource *r = to_descriptor_resource(resource);
 
 922
 923	fw_core_remove_descriptor(&r->descriptor);
 924	kfree(r);
 925}
 926
 927static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg)
 928{
 929	struct fw_cdev_add_descriptor *a = &arg->add_descriptor;
 930	struct descriptor_resource *r;
 931	int ret;
 932
 933	/* Access policy: Allow this ioctl only on local nodes' device files. */
 934	if (!client->device->is_local)
 935		return -ENOSYS;
 936
 937	if (a->length > 256)
 938		return -EINVAL;
 939
 940	r = kmalloc(sizeof(*r) + a->length * 4, GFP_KERNEL);
 941	if (r == NULL)
 942		return -ENOMEM;
 943
 944	if (copy_from_user(r->data, u64_to_uptr(a->data), a->length * 4)) {
 945		ret = -EFAULT;
 946		goto failed;
 947	}
 948
 949	r->descriptor.length    = a->length;
 950	r->descriptor.immediate = a->immediate;
 951	r->descriptor.key       = a->key;
 952	r->descriptor.data      = r->data;
 953
 954	ret = fw_core_add_descriptor(&r->descriptor);
 955	if (ret < 0)
 956		goto failed;
 957
 958	r->resource.release = release_descriptor;
 959	ret = add_client_resource(client, &r->resource, GFP_KERNEL);
 960	if (ret < 0) {
 961		fw_core_remove_descriptor(&r->descriptor);
 962		goto failed;
 963	}
 964	a->handle = r->resource.handle;
 965
 966	return 0;
 967 failed:
 968	kfree(r);
 969
 970	return ret;
 971}
 972
 973static int ioctl_remove_descriptor(struct client *client, union ioctl_arg *arg)
 974{
 975	return release_client_resource(client, arg->remove_descriptor.handle,
 976				       release_descriptor, NULL);
 977}
 978
 979static void iso_callback(struct fw_iso_context *context, u32 cycle,
 980			 size_t header_length, void *header, void *data)
 981{
 982	struct client *client = data;
 983	struct iso_interrupt_event *e;
 984
 985	e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL);
 986	if (e == NULL)
 987		return;
 988
 989	e->interrupt.type      = FW_CDEV_EVENT_ISO_INTERRUPT;
 990	e->interrupt.closure   = client->iso_closure;
 991	e->interrupt.cycle     = cycle;
 992	e->interrupt.header_length = header_length;
 993	memcpy(e->interrupt.header, header, header_length);
 994	queue_event(client, &e->event, &e->interrupt,
 995		    sizeof(e->interrupt) + header_length, NULL, 0);
 996}
 997
 998static void iso_mc_callback(struct fw_iso_context *context,
 999			    dma_addr_t completed, void *data)
1000{
1001	struct client *client = data;
1002	struct iso_interrupt_mc_event *e;
1003
1004	e = kmalloc(sizeof(*e), GFP_KERNEL);
1005	if (e == NULL)
1006		return;
1007
1008	e->interrupt.type      = FW_CDEV_EVENT_ISO_INTERRUPT_MULTICHANNEL;
1009	e->interrupt.closure   = client->iso_closure;
1010	e->interrupt.completed = fw_iso_buffer_lookup(&client->buffer,
1011						      completed);
1012	queue_event(client, &e->event, &e->interrupt,
1013		    sizeof(e->interrupt), NULL, 0);
1014}
1015
1016static enum dma_data_direction iso_dma_direction(struct fw_iso_context *context)
1017{
1018		if (context->type == FW_ISO_CONTEXT_TRANSMIT)
1019			return DMA_TO_DEVICE;
1020		else
1021			return DMA_FROM_DEVICE;
1022}
1023
1024static struct fw_iso_context *fw_iso_mc_context_create(struct fw_card *card,
1025						fw_iso_mc_callback_t callback,
1026						void *callback_data)
1027{
1028	struct fw_iso_context *ctx;
1029
1030	ctx = fw_iso_context_create(card, FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL,
1031				    0, 0, 0, NULL, callback_data);
1032	if (!IS_ERR(ctx))
1033		ctx->callback.mc = callback;
1034
1035	return ctx;
1036}
1037
1038static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)
1039{
1040	struct fw_cdev_create_iso_context *a = &arg->create_iso_context;
1041	struct fw_iso_context *context;
1042	union fw_iso_callback cb;
1043	int ret;
1044
1045	BUILD_BUG_ON(FW_CDEV_ISO_CONTEXT_TRANSMIT != FW_ISO_CONTEXT_TRANSMIT ||
1046		     FW_CDEV_ISO_CONTEXT_RECEIVE  != FW_ISO_CONTEXT_RECEIVE  ||
1047		     FW_CDEV_ISO_CONTEXT_RECEIVE_MULTICHANNEL !=
1048					FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL);
1049
1050	switch (a->type) {
1051	case FW_ISO_CONTEXT_TRANSMIT:
1052		if (a->speed > SCODE_3200 || a->channel > 63)
1053			return -EINVAL;
1054
1055		cb.sc = iso_callback;
1056		break;
1057
1058	case FW_ISO_CONTEXT_RECEIVE:
1059		if (a->header_size < 4 || (a->header_size & 3) ||
1060		    a->channel > 63)
1061			return -EINVAL;
1062
1063		cb.sc = iso_callback;
1064		break;
1065
1066	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
1067		cb.mc = iso_mc_callback;
1068		break;
1069
1070	default:
1071		return -EINVAL;
1072	}
1073
1074	if (a->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL)
1075		context = fw_iso_mc_context_create(client->device->card, cb.mc,
1076						   client);
1077	else
1078		context = fw_iso_context_create(client->device->card, a->type,
1079						a->channel, a->speed,
1080						a->header_size, cb.sc, client);
1081	if (IS_ERR(context))
1082		return PTR_ERR(context);
1083	if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW)
1084		context->drop_overflow_headers = true;
1085
1086	// We only support one context at this time.
1087	guard(spinlock_irq)(&client->lock);
1088
1089	if (client->iso_context != NULL) {
 
1090		fw_iso_context_destroy(context);
1091
1092		return -EBUSY;
1093	}
1094	if (!client->buffer_is_mapped) {
1095		ret = fw_iso_buffer_map_dma(&client->buffer,
1096					    client->device->card,
1097					    iso_dma_direction(context));
1098		if (ret < 0) {
 
1099			fw_iso_context_destroy(context);
1100
1101			return ret;
1102		}
1103		client->buffer_is_mapped = true;
1104	}
1105	client->iso_closure = a->closure;
1106	client->iso_context = context;
 
1107
1108	a->handle = 0;
1109
1110	return 0;
1111}
1112
1113static int ioctl_set_iso_channels(struct client *client, union ioctl_arg *arg)
1114{
1115	struct fw_cdev_set_iso_channels *a = &arg->set_iso_channels;
1116	struct fw_iso_context *ctx = client->iso_context;
1117
1118	if (ctx == NULL || a->handle != 0)
1119		return -EINVAL;
1120
1121	return fw_iso_context_set_channels(ctx, &a->channels);
1122}
1123
1124/* Macros for decoding the iso packet control header. */
1125#define GET_PAYLOAD_LENGTH(v)	((v) & 0xffff)
1126#define GET_INTERRUPT(v)	(((v) >> 16) & 0x01)
1127#define GET_SKIP(v)		(((v) >> 17) & 0x01)
1128#define GET_TAG(v)		(((v) >> 18) & 0x03)
1129#define GET_SY(v)		(((v) >> 20) & 0x0f)
1130#define GET_HEADER_LENGTH(v)	(((v) >> 24) & 0xff)
1131
1132static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)
1133{
1134	struct fw_cdev_queue_iso *a = &arg->queue_iso;
1135	struct fw_cdev_iso_packet __user *p, *end, *next;
1136	struct fw_iso_context *ctx = client->iso_context;
1137	unsigned long payload, buffer_end, transmit_header_bytes = 0;
1138	u32 control;
1139	int count;
1140	struct {
1141		struct fw_iso_packet packet;
1142		u8 header[256];
1143	} u;
1144
1145	if (ctx == NULL || a->handle != 0)
1146		return -EINVAL;
1147
1148	/*
1149	 * If the user passes a non-NULL data pointer, has mmap()'ed
1150	 * the iso buffer, and the pointer points inside the buffer,
1151	 * we setup the payload pointers accordingly.  Otherwise we
1152	 * set them both to 0, which will still let packets with
1153	 * payload_length == 0 through.  In other words, if no packets
1154	 * use the indirect payload, the iso buffer need not be mapped
1155	 * and the a->data pointer is ignored.
1156	 */
1157	payload = (unsigned long)a->data - client->vm_start;
1158	buffer_end = client->buffer.page_count << PAGE_SHIFT;
1159	if (a->data == 0 || client->buffer.pages == NULL ||
1160	    payload >= buffer_end) {
1161		payload = 0;
1162		buffer_end = 0;
1163	}
1164
1165	if (ctx->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL && payload & 3)
1166		return -EINVAL;
1167
1168	p = (struct fw_cdev_iso_packet __user *)u64_to_uptr(a->packets);
 
 
1169
1170	end = (void __user *)p + a->size;
1171	count = 0;
1172	while (p < end) {
1173		if (get_user(control, &p->control))
1174			return -EFAULT;
1175		u.packet.payload_length = GET_PAYLOAD_LENGTH(control);
1176		u.packet.interrupt = GET_INTERRUPT(control);
1177		u.packet.skip = GET_SKIP(control);
1178		u.packet.tag = GET_TAG(control);
1179		u.packet.sy = GET_SY(control);
1180		u.packet.header_length = GET_HEADER_LENGTH(control);
1181
1182		switch (ctx->type) {
1183		case FW_ISO_CONTEXT_TRANSMIT:
1184			if (u.packet.header_length & 3)
1185				return -EINVAL;
1186			transmit_header_bytes = u.packet.header_length;
1187			break;
1188
1189		case FW_ISO_CONTEXT_RECEIVE:
1190			if (u.packet.header_length == 0 ||
1191			    u.packet.header_length % ctx->header_size != 0)
1192				return -EINVAL;
1193			break;
1194
1195		case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
1196			if (u.packet.payload_length == 0 ||
1197			    u.packet.payload_length & 3)
1198				return -EINVAL;
1199			break;
1200		}
1201
1202		next = (struct fw_cdev_iso_packet __user *)
1203			&p->header[transmit_header_bytes / 4];
1204		if (next > end)
1205			return -EINVAL;
1206		if (copy_from_user
1207		    (u.packet.header, p->header, transmit_header_bytes))
1208			return -EFAULT;
1209		if (u.packet.skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT &&
1210		    u.packet.header_length + u.packet.payload_length > 0)
1211			return -EINVAL;
1212		if (payload + u.packet.payload_length > buffer_end)
1213			return -EINVAL;
1214
1215		if (fw_iso_context_queue(ctx, &u.packet,
1216					 &client->buffer, payload))
1217			break;
1218
1219		p = next;
1220		payload += u.packet.payload_length;
1221		count++;
1222	}
1223	fw_iso_context_queue_flush(ctx);
1224
1225	a->size    -= uptr_to_u64(p) - a->packets;
1226	a->packets  = uptr_to_u64(p);
1227	a->data     = client->vm_start + payload;
1228
1229	return count;
1230}
1231
1232static int ioctl_start_iso(struct client *client, union ioctl_arg *arg)
1233{
1234	struct fw_cdev_start_iso *a = &arg->start_iso;
1235
1236	BUILD_BUG_ON(
1237	    FW_CDEV_ISO_CONTEXT_MATCH_TAG0 != FW_ISO_CONTEXT_MATCH_TAG0 ||
1238	    FW_CDEV_ISO_CONTEXT_MATCH_TAG1 != FW_ISO_CONTEXT_MATCH_TAG1 ||
1239	    FW_CDEV_ISO_CONTEXT_MATCH_TAG2 != FW_ISO_CONTEXT_MATCH_TAG2 ||
1240	    FW_CDEV_ISO_CONTEXT_MATCH_TAG3 != FW_ISO_CONTEXT_MATCH_TAG3 ||
1241	    FW_CDEV_ISO_CONTEXT_MATCH_ALL_TAGS != FW_ISO_CONTEXT_MATCH_ALL_TAGS);
1242
1243	if (client->iso_context == NULL || a->handle != 0)
1244		return -EINVAL;
1245
1246	if (client->iso_context->type == FW_ISO_CONTEXT_RECEIVE &&
1247	    (a->tags == 0 || a->tags > 15 || a->sync > 15))
1248		return -EINVAL;
1249
1250	return fw_iso_context_start(client->iso_context,
1251				    a->cycle, a->sync, a->tags);
1252}
1253
1254static int ioctl_stop_iso(struct client *client, union ioctl_arg *arg)
1255{
1256	struct fw_cdev_stop_iso *a = &arg->stop_iso;
1257
1258	if (client->iso_context == NULL || a->handle != 0)
1259		return -EINVAL;
1260
1261	return fw_iso_context_stop(client->iso_context);
1262}
1263
1264static int ioctl_flush_iso(struct client *client, union ioctl_arg *arg)
1265{
1266	struct fw_cdev_flush_iso *a = &arg->flush_iso;
1267
1268	if (client->iso_context == NULL || a->handle != 0)
1269		return -EINVAL;
1270
1271	return fw_iso_context_flush_completions(client->iso_context);
1272}
1273
1274static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg)
1275{
1276	struct fw_cdev_get_cycle_timer2 *a = &arg->get_cycle_timer2;
1277	struct fw_card *card = client->device->card;
1278	struct timespec64 ts = {0, 0};
1279	u32 cycle_time = 0;
1280	int ret;
1281
1282	guard(irq)();
1283
1284	ret = fw_card_read_cycle_time(card, &cycle_time);
1285	if (ret < 0)
1286		return ret;
1287
1288	switch (a->clk_id) {
1289	case CLOCK_REALTIME:      ktime_get_real_ts64(&ts);	break;
1290	case CLOCK_MONOTONIC:     ktime_get_ts64(&ts);		break;
1291	case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts);	break;
1292	default:
1293		return -EINVAL;
1294	}
1295
 
 
1296	a->tv_sec      = ts.tv_sec;
1297	a->tv_nsec     = ts.tv_nsec;
1298	a->cycle_timer = cycle_time;
1299
1300	return 0;
1301}
1302
1303static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)
1304{
1305	struct fw_cdev_get_cycle_timer *a = &arg->get_cycle_timer;
1306	struct fw_cdev_get_cycle_timer2 ct2;
1307
1308	ct2.clk_id = CLOCK_REALTIME;
1309	ioctl_get_cycle_timer2(client, (union ioctl_arg *)&ct2);
1310
1311	a->local_time = ct2.tv_sec * USEC_PER_SEC + ct2.tv_nsec / NSEC_PER_USEC;
1312	a->cycle_timer = ct2.cycle_timer;
1313
1314	return 0;
1315}
1316
1317static void iso_resource_work(struct work_struct *work)
1318{
1319	struct iso_resource_event *e;
1320	struct iso_resource *r =
1321			container_of(work, struct iso_resource, work.work);
1322	struct client *client = r->client;
1323	unsigned long index = r->resource.handle;
1324	int generation, channel, bandwidth, todo;
1325	bool skip, free, success;
1326
1327	scoped_guard(spinlock_irq, &client->lock) {
1328		generation = client->device->generation;
1329		todo = r->todo;
1330		// Allow 1000ms grace period for other reallocations.
1331		if (todo == ISO_RES_ALLOC &&
1332		    time_before64(get_jiffies_64(), client->device->card->reset_jiffies + HZ)) {
1333			schedule_iso_resource(r, DIV_ROUND_UP(HZ, 3));
1334			skip = true;
1335		} else {
1336			// We could be called twice within the same generation.
1337			skip = todo == ISO_RES_REALLOC &&
1338			       r->generation == generation;
1339		}
1340		free = todo == ISO_RES_DEALLOC ||
1341		       todo == ISO_RES_ALLOC_ONCE ||
1342		       todo == ISO_RES_DEALLOC_ONCE;
1343		r->generation = generation;
1344	}
 
1345
1346	if (skip)
1347		goto out;
1348
1349	bandwidth = r->bandwidth;
1350
1351	fw_iso_resource_manage(client->device->card, generation,
1352			r->channels, &channel, &bandwidth,
1353			todo == ISO_RES_ALLOC ||
1354			todo == ISO_RES_REALLOC ||
1355			todo == ISO_RES_ALLOC_ONCE);
1356	/*
1357	 * Is this generation outdated already?  As long as this resource sticks
1358	 * in the xarray, it will be scheduled again for a newer generation or at
1359	 * shutdown.
1360	 */
1361	if (channel == -EAGAIN &&
1362	    (todo == ISO_RES_ALLOC || todo == ISO_RES_REALLOC))
1363		goto out;
1364
1365	success = channel >= 0 || bandwidth > 0;
1366
1367	scoped_guard(spinlock_irq, &client->lock) {
1368		// Transit from allocation to reallocation, except if the client
1369		// requested deallocation in the meantime.
1370		if (r->todo == ISO_RES_ALLOC)
1371			r->todo = ISO_RES_REALLOC;
1372		// Allocation or reallocation failure?  Pull this resource out of the
1373		// xarray and prepare for deletion, unless the client is shutting down.
1374		if (r->todo == ISO_RES_REALLOC && !success &&
1375		    !client->in_shutdown &&
1376		    xa_erase(&client->resource_xa, index)) {
1377			client_put(client);
1378			free = true;
1379		}
 
 
 
1380	}
 
1381
1382	if (todo == ISO_RES_ALLOC && channel >= 0)
1383		r->channels = 1ULL << channel;
1384
1385	if (todo == ISO_RES_REALLOC && success)
1386		goto out;
1387
1388	if (todo == ISO_RES_ALLOC || todo == ISO_RES_ALLOC_ONCE) {
1389		e = r->e_alloc;
1390		r->e_alloc = NULL;
1391	} else {
1392		e = r->e_dealloc;
1393		r->e_dealloc = NULL;
1394	}
1395	e->iso_resource.handle    = r->resource.handle;
1396	e->iso_resource.channel   = channel;
1397	e->iso_resource.bandwidth = bandwidth;
1398
1399	queue_event(client, &e->event,
1400		    &e->iso_resource, sizeof(e->iso_resource), NULL, 0);
1401
1402	if (free) {
1403		cancel_delayed_work(&r->work);
1404		kfree(r->e_alloc);
1405		kfree(r->e_dealloc);
1406		kfree(r);
1407	}
1408 out:
1409	client_put(client);
1410}
1411
1412static void release_iso_resource(struct client *client,
1413				 struct client_resource *resource)
1414{
1415	struct iso_resource *r = to_iso_resource(resource);
1416
1417	guard(spinlock_irq)(&client->lock);
1418
 
1419	r->todo = ISO_RES_DEALLOC;
1420	schedule_iso_resource(r, 0);
 
1421}
1422
1423static int init_iso_resource(struct client *client,
1424		struct fw_cdev_allocate_iso_resource *request, int todo)
1425{
1426	struct iso_resource_event *e1, *e2;
1427	struct iso_resource *r;
1428	int ret;
1429
1430	if ((request->channels == 0 && request->bandwidth == 0) ||
1431	    request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL)
1432		return -EINVAL;
1433
1434	r  = kmalloc(sizeof(*r), GFP_KERNEL);
1435	e1 = kmalloc(sizeof(*e1), GFP_KERNEL);
1436	e2 = kmalloc(sizeof(*e2), GFP_KERNEL);
1437	if (r == NULL || e1 == NULL || e2 == NULL) {
1438		ret = -ENOMEM;
1439		goto fail;
1440	}
1441
1442	INIT_DELAYED_WORK(&r->work, iso_resource_work);
1443	r->client	= client;
1444	r->todo		= todo;
1445	r->generation	= -1;
1446	r->channels	= request->channels;
1447	r->bandwidth	= request->bandwidth;
1448	r->e_alloc	= e1;
1449	r->e_dealloc	= e2;
1450
1451	e1->iso_resource.closure = request->closure;
1452	e1->iso_resource.type    = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
1453	e2->iso_resource.closure = request->closure;
1454	e2->iso_resource.type    = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
1455
1456	if (todo == ISO_RES_ALLOC) {
1457		r->resource.release = release_iso_resource;
1458		ret = add_client_resource(client, &r->resource, GFP_KERNEL);
1459		if (ret < 0)
1460			goto fail;
1461	} else {
1462		r->resource.release = NULL;
1463		r->resource.handle = -1;
1464		schedule_iso_resource(r, 0);
1465	}
1466	request->handle = r->resource.handle;
1467
1468	return 0;
1469 fail:
1470	kfree(r);
1471	kfree(e1);
1472	kfree(e2);
1473
1474	return ret;
1475}
1476
1477static int ioctl_allocate_iso_resource(struct client *client,
1478				       union ioctl_arg *arg)
1479{
1480	return init_iso_resource(client,
1481			&arg->allocate_iso_resource, ISO_RES_ALLOC);
1482}
1483
1484static int ioctl_deallocate_iso_resource(struct client *client,
1485					 union ioctl_arg *arg)
1486{
1487	return release_client_resource(client,
1488			arg->deallocate.handle, release_iso_resource, NULL);
1489}
1490
1491static int ioctl_allocate_iso_resource_once(struct client *client,
1492					    union ioctl_arg *arg)
1493{
1494	return init_iso_resource(client,
1495			&arg->allocate_iso_resource, ISO_RES_ALLOC_ONCE);
1496}
1497
1498static int ioctl_deallocate_iso_resource_once(struct client *client,
1499					      union ioctl_arg *arg)
1500{
1501	return init_iso_resource(client,
1502			&arg->allocate_iso_resource, ISO_RES_DEALLOC_ONCE);
1503}
1504
1505/*
1506 * Returns a speed code:  Maximum speed to or from this device,
1507 * limited by the device's link speed, the local node's link speed,
1508 * and all PHY port speeds between the two links.
1509 */
1510static int ioctl_get_speed(struct client *client, union ioctl_arg *arg)
1511{
1512	return client->device->max_speed;
1513}
1514
1515static int ioctl_send_broadcast_request(struct client *client,
1516					union ioctl_arg *arg)
1517{
1518	struct fw_cdev_send_request *a = &arg->send_request;
1519
1520	switch (a->tcode) {
1521	case TCODE_WRITE_QUADLET_REQUEST:
1522	case TCODE_WRITE_BLOCK_REQUEST:
1523		break;
1524	default:
1525		return -EINVAL;
1526	}
1527
1528	/* Security policy: Only allow accesses to Units Space. */
1529	if (a->offset < CSR_REGISTER_BASE + CSR_CONFIG_ROM_END)
1530		return -EACCES;
1531
1532	return init_request(client, a, LOCAL_BUS | 0x3f, SCODE_100);
1533}
1534
1535static int ioctl_send_stream_packet(struct client *client, union ioctl_arg *arg)
1536{
1537	struct fw_cdev_send_stream_packet *a = &arg->send_stream_packet;
1538	struct fw_cdev_send_request request;
1539	int dest;
1540
1541	if (a->speed > client->device->card->link_speed ||
1542	    a->length > 1024 << a->speed)
1543		return -EIO;
1544
1545	if (a->tag > 3 || a->channel > 63 || a->sy > 15)
1546		return -EINVAL;
1547
1548	dest = fw_stream_packet_destination_id(a->tag, a->channel, a->sy);
1549	request.tcode		= TCODE_STREAM_DATA;
1550	request.length		= a->length;
1551	request.closure		= a->closure;
1552	request.data		= a->data;
1553	request.generation	= a->generation;
1554
1555	return init_request(client, &request, dest, a->speed);
1556}
1557
1558static void outbound_phy_packet_callback(struct fw_packet *packet,
1559					 struct fw_card *card, int status)
1560{
1561	struct outbound_phy_packet_event *e =
1562		container_of(packet, struct outbound_phy_packet_event, p);
1563	struct client *e_client = e->client;
1564	u32 rcode;
1565
1566	trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation,
1567					  packet->timestamp);
1568
1569	switch (status) {
1570	// expected:
1571	case ACK_COMPLETE:
1572		rcode = RCODE_COMPLETE;
1573		break;
1574	// should never happen with PHY packets:
1575	case ACK_PENDING:
1576		rcode = RCODE_COMPLETE;
1577		break;
1578	case ACK_BUSY_X:
1579	case ACK_BUSY_A:
1580	case ACK_BUSY_B:
1581		rcode = RCODE_BUSY;
1582		break;
1583	case ACK_DATA_ERROR:
1584		rcode = RCODE_DATA_ERROR;
1585		break;
1586	case ACK_TYPE_ERROR:
1587		rcode = RCODE_TYPE_ERROR;
1588		break;
1589	// stale generation; cancelled; on certain controllers: no ack
1590	default:
1591		rcode = status;
1592		break;
1593	}
1594
1595	switch (e->phy_packet.without_tstamp.type) {
1596	case FW_CDEV_EVENT_PHY_PACKET_SENT:
1597	{
1598		struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1599
1600		pp->rcode = rcode;
1601		pp->data[0] = packet->timestamp;
1602		queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
1603			    NULL, 0);
1604		break;
1605	}
1606	case FW_CDEV_EVENT_PHY_PACKET_SENT2:
1607	{
1608		struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1609
1610		pp->rcode = rcode;
1611		pp->tstamp = packet->timestamp;
1612		queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
1613			    NULL, 0);
1614		break;
1615	}
1616	default:
1617		WARN_ON(1);
1618		break;
1619	}
1620
1621	client_put(e_client);
1622}
1623
1624static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
1625{
1626	struct fw_cdev_send_phy_packet *a = &arg->send_phy_packet;
1627	struct fw_card *card = client->device->card;
1628	struct outbound_phy_packet_event *e;
1629
1630	/* Access policy: Allow this ioctl only on local nodes' device files. */
1631	if (!client->device->is_local)
1632		return -ENOSYS;
1633
1634	e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL);
1635	if (e == NULL)
1636		return -ENOMEM;
1637
1638	client_get(client);
1639	e->client		= client;
1640	e->p.speed		= SCODE_100;
1641	e->p.generation		= a->generation;
1642	async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL);
1643	e->p.header[1]		= a->data[0];
1644	e->p.header[2]		= a->data[1];
1645	e->p.header_length	= 12;
1646	e->p.callback		= outbound_phy_packet_callback;
1647
1648	if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
1649		struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1650
1651		pp->closure = a->closure;
1652		pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT;
1653		if (is_ping_packet(a->data))
1654			pp->length = 4;
1655	} else {
1656		struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1657
1658		pp->closure = a->closure;
1659		pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2;
1660		// Keep the data field so that application can match the response event to the
1661		// request.
1662		pp->length = sizeof(a->data);
1663		memcpy(pp->data, a->data, sizeof(a->data));
1664	}
1665
1666	trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation,
1667					  e->p.header[1], e->p.header[2]);
1668
1669	card->driver->send_request(card, &e->p);
1670
1671	return 0;
1672}
1673
1674static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg)
1675{
1676	struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets;
1677	struct fw_card *card = client->device->card;
1678
1679	/* Access policy: Allow this ioctl only on local nodes' device files. */
1680	if (!client->device->is_local)
1681		return -ENOSYS;
1682
1683	guard(spinlock_irq)(&card->lock);
1684
1685	list_move_tail(&client->phy_receiver_link, &card->phy_receiver_list);
1686	client->phy_receiver_closure = a->closure;
1687
 
 
1688	return 0;
1689}
1690
1691void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p)
1692{
1693	struct client *client;
 
 
1694
1695	guard(spinlock_irqsave)(&card->lock);
1696
1697	list_for_each_entry(client, &card->phy_receiver_list, phy_receiver_link) {
1698		struct inbound_phy_packet_event *e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC);
1699		if (e == NULL)
1700			break;
1701
1702		if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
1703			struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1704
1705			pp->closure = client->phy_receiver_closure;
1706			pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED;
1707			pp->rcode = RCODE_COMPLETE;
1708			pp->length = 8;
1709			pp->data[0] = p->header[1];
1710			pp->data[1] = p->header[2];
1711			queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
1712		} else {
1713			struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1714
1715			pp = &e->phy_packet.with_tstamp;
1716			pp->closure = client->phy_receiver_closure;
1717			pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2;
1718			pp->rcode = RCODE_COMPLETE;
1719			pp->length = 8;
1720			pp->tstamp = p->timestamp;
1721			pp->data[0] = p->header[1];
1722			pp->data[1] = p->header[2];
1723			queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
1724		}
1725	}
 
 
1726}
1727
1728static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = {
1729	[0x00] = ioctl_get_info,
1730	[0x01] = ioctl_send_request,
1731	[0x02] = ioctl_allocate,
1732	[0x03] = ioctl_deallocate,
1733	[0x04] = ioctl_send_response,
1734	[0x05] = ioctl_initiate_bus_reset,
1735	[0x06] = ioctl_add_descriptor,
1736	[0x07] = ioctl_remove_descriptor,
1737	[0x08] = ioctl_create_iso_context,
1738	[0x09] = ioctl_queue_iso,
1739	[0x0a] = ioctl_start_iso,
1740	[0x0b] = ioctl_stop_iso,
1741	[0x0c] = ioctl_get_cycle_timer,
1742	[0x0d] = ioctl_allocate_iso_resource,
1743	[0x0e] = ioctl_deallocate_iso_resource,
1744	[0x0f] = ioctl_allocate_iso_resource_once,
1745	[0x10] = ioctl_deallocate_iso_resource_once,
1746	[0x11] = ioctl_get_speed,
1747	[0x12] = ioctl_send_broadcast_request,
1748	[0x13] = ioctl_send_stream_packet,
1749	[0x14] = ioctl_get_cycle_timer2,
1750	[0x15] = ioctl_send_phy_packet,
1751	[0x16] = ioctl_receive_phy_packets,
1752	[0x17] = ioctl_set_iso_channels,
1753	[0x18] = ioctl_flush_iso,
1754};
1755
1756static int dispatch_ioctl(struct client *client,
1757			  unsigned int cmd, void __user *arg)
1758{
1759	union ioctl_arg buffer;
1760	int ret;
1761
1762	if (fw_device_is_shutdown(client->device))
1763		return -ENODEV;
1764
1765	if (_IOC_TYPE(cmd) != '#' ||
1766	    _IOC_NR(cmd) >= ARRAY_SIZE(ioctl_handlers) ||
1767	    _IOC_SIZE(cmd) > sizeof(buffer))
1768		return -ENOTTY;
1769
1770	memset(&buffer, 0, sizeof(buffer));
1771
1772	if (_IOC_DIR(cmd) & _IOC_WRITE)
1773		if (copy_from_user(&buffer, arg, _IOC_SIZE(cmd)))
1774			return -EFAULT;
1775
1776	ret = ioctl_handlers[_IOC_NR(cmd)](client, &buffer);
1777	if (ret < 0)
1778		return ret;
1779
1780	if (_IOC_DIR(cmd) & _IOC_READ)
1781		if (copy_to_user(arg, &buffer, _IOC_SIZE(cmd)))
1782			return -EFAULT;
1783
1784	return ret;
1785}
1786
1787static long fw_device_op_ioctl(struct file *file,
1788			       unsigned int cmd, unsigned long arg)
1789{
1790	return dispatch_ioctl(file->private_data, cmd, (void __user *)arg);
1791}
1792
 
 
 
 
 
 
 
 
1793static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma)
1794{
1795	struct client *client = file->private_data;
1796	unsigned long size;
1797	int page_count, ret;
1798
1799	if (fw_device_is_shutdown(client->device))
1800		return -ENODEV;
1801
1802	/* FIXME: We could support multiple buffers, but we don't. */
1803	if (client->buffer.pages != NULL)
1804		return -EBUSY;
1805
1806	if (!(vma->vm_flags & VM_SHARED))
1807		return -EINVAL;
1808
1809	if (vma->vm_start & ~PAGE_MASK)
1810		return -EINVAL;
1811
1812	client->vm_start = vma->vm_start;
1813	size = vma->vm_end - vma->vm_start;
1814	page_count = size >> PAGE_SHIFT;
1815	if (size & ~PAGE_MASK)
1816		return -EINVAL;
1817
1818	ret = fw_iso_buffer_alloc(&client->buffer, page_count);
1819	if (ret < 0)
1820		return ret;
1821
1822	scoped_guard(spinlock_irq, &client->lock) {
1823		if (client->iso_context) {
1824			ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,
1825						    iso_dma_direction(client->iso_context));
1826			if (ret < 0)
1827				goto fail;
1828			client->buffer_is_mapped = true;
1829		}
1830	}
 
 
 
1831
1832	ret = vm_map_pages_zero(vma, client->buffer.pages,
1833				client->buffer.page_count);
1834	if (ret < 0)
1835		goto fail;
1836
1837	return 0;
1838 fail:
1839	fw_iso_buffer_destroy(&client->buffer, client->device->card);
1840	return ret;
1841}
1842
1843static bool has_outbound_transactions(struct client *client)
1844{
1845	struct client_resource *resource;
1846	unsigned long index;
1847
1848	guard(spinlock_irq)(&client->lock);
 
1849
1850	xa_for_each(&client->resource_xa, index, resource) {
1851		if (is_outbound_transaction_resource(resource))
1852			return true;
1853	}
 
 
 
 
1854
1855	return false;
 
 
 
 
 
 
 
 
 
 
 
1856}
1857
1858static int fw_device_op_release(struct inode *inode, struct file *file)
1859{
1860	struct client *client = file->private_data;
1861	struct event *event, *next_event;
1862	struct client_resource *resource;
1863	unsigned long index;
1864
1865	scoped_guard(spinlock_irq, &client->device->card->lock)
1866		list_del(&client->phy_receiver_link);
1867
1868	scoped_guard(mutex, &client->device->client_list_mutex)
1869		list_del(&client->link);
 
 
1870
1871	if (client->iso_context)
1872		fw_iso_context_destroy(client->iso_context);
1873
1874	if (client->buffer.pages)
1875		fw_iso_buffer_destroy(&client->buffer, client->device->card);
1876
1877	// Freeze client->resource_xa and client->event_list.
1878	scoped_guard(spinlock_irq, &client->lock)
1879		client->in_shutdown = true;
 
1880
1881	wait_event(client->tx_flush_wait, !has_outbound_transactions(client));
1882
1883	xa_for_each(&client->resource_xa, index, resource) {
1884		resource->release(client, resource);
1885		client_put(client);
1886	}
1887	xa_destroy(&client->resource_xa);
1888
1889	list_for_each_entry_safe(event, next_event, &client->event_list, link)
1890		kfree(event);
1891
1892	client_put(client);
1893
1894	return 0;
1895}
1896
1897static __poll_t fw_device_op_poll(struct file *file, poll_table * pt)
1898{
1899	struct client *client = file->private_data;
1900	__poll_t mask = 0;
1901
1902	poll_wait(file, &client->wait, pt);
1903
1904	if (fw_device_is_shutdown(client->device))
1905		mask |= EPOLLHUP | EPOLLERR;
1906	if (!list_empty(&client->event_list))
1907		mask |= EPOLLIN | EPOLLRDNORM;
1908
1909	return mask;
1910}
1911
1912const struct file_operations fw_device_ops = {
1913	.owner		= THIS_MODULE,
 
1914	.open		= fw_device_op_open,
1915	.read		= fw_device_op_read,
1916	.unlocked_ioctl	= fw_device_op_ioctl,
1917	.mmap		= fw_device_op_mmap,
1918	.release	= fw_device_op_release,
1919	.poll		= fw_device_op_poll,
1920	.compat_ioctl	= compat_ptr_ioctl,
 
 
1921};